diff --git a/.gitignore b/.gitignore index 0839cf48..646c1d68 100755 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,11 @@ assets/favorites.json assets/tags.json public/changelog public/garden -public/licenses.txt \ No newline at end of file +public/licenses.txt + +syndications/cache +syndications/indiekit_token.json +syndications/liked_songs.json +syndications/media_urls.json +syndications/youtube_credentials.json +syndications/youtube_token.json diff --git a/bun.lockb b/bun.lockb index 1d5f0a83..7c5ef361 100644 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/nuxt.config.ts b/nuxt.config.ts index d456838a..d07dddde 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -152,6 +152,10 @@ export default async () => { }, { rel: 'token_endpoint', href: 'https://indie.incremental.social/auth/token' }, { rel: 'micropub', href: 'https://indie.incremental.social/micropub' }, + { + rel: 'indieauth-metadata', + href: 'https://indie.incremental.social/.well-known/oauth-authorization-server' + }, /** Feeds */ { ...feedProps, href: '/posts', title: 'All posts' }, diff --git a/package.json b/package.json index 9b1c0a11..a00822f7 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "serve": "bun run ./.output/server/index.mjs", "logseq-export": "run-script-os", "logseq-export:win32": ".\\logseq-export\\logseq-export.exe --logseqFolder ./Garden --outputFolder ./garden-output", - "logseq-export:linux": "chmod +x logseq-export/logseq-export && logseq-export/logseq-export --logseqFolder ./Garden --outputFolder ./garden-output" + "logseq-export:linux": "chmod +x logseq-export/logseq-export && logseq-export/logseq-export --logseqFolder ./Garden --outputFolder ./garden-output", + "update:youtube": "bun run syndications/update_youtube.ts" }, "dependencies": { "@iconify-json/cib": "^1.2.0", @@ -28,8 +29,13 @@ "sass-embedded": "^1.79.2" }, "devDependencies": { + "@types/bun": "^1.1.10", + "dotenv": "^16.4.5", "feed": "^4.2.2", + "file-type": "^19.5.0", "generate-license-file": "^3.5.1", + "googleapis": "^144.0.0", + "open": "^10.1.0", "run-script-os": "^1.1.6", "word-counting": "^1.1.4" } diff --git a/syndications/add_youtube_reply.ts b/syndications/add_youtube_reply.ts new file mode 100644 index 00000000..6481524b --- /dev/null +++ b/syndications/add_youtube_reply.ts @@ -0,0 +1,79 @@ +import { addReply } from "./indiekit"; +import { authorize, getBestThumbnail, getComment, getVideos, VIDEO_CATEGORIES } from "./youtube_utils"; + +let commentId: string, videoId: string; + +async function run() { + console.log("Enter the reply to add here:"); + for await (const url of console as unknown as AsyncIterable) { + const [_, video, comment] = url.match( + /https?:\/\/(?:www\.)?youtube.com\/watch\?v=([0-9a-zA-Z_\.-]+)&lc=([0-9a-zA-Z_\.-]+)/ + ) as string[]; + + if (!video || !comment) { + console.error("Could not parse video and comment id from url", url); + } else { + commentId = comment; + videoId = video; + break; + } + } +} + +run() +.then(authorize) +.then(async auth => { + const url = `https://www.youtube.com/watch?v=${videoId}&lc=${commentId}`; + const comment = (await getComment(commentId, auth))?.[0]; + const video = (await getVideos(videoId, auth))?.[0]; + if (!comment || !video) { + console.log("Couldn't retrieve either the comment or video", comment, video); + return; + } + + const parentComment = comment.parentId == null ? undefined : + (await getComment(comment.parentId, auth))?.[0]; + + const category = VIDEO_CATEGORIES[parseInt(video.snippet.categoryId ?? "1")]; + await addReply({ + "in-reply-to": parentComment ? + `https://www.youtube.com/watch?v=${videoId}&lc=${comment.parentId!}` : + `https://www.youtube.com/watch?v=${videoId}`, + content: comment.textDisplay ?? "", + published: comment.publishedAt == null ? undefined : new Date(comment.publishedAt), + category, + originalUrl: url, + parent: parentComment ? { + kind: "reply", + description: parentComment.textDisplay ?? "", + syndications: [ + `https://www.youtube.com/watch?v=${videoId}&lc=${comment.parentId!}` + ], + tags: [category], + published: parentComment.publishedAt == null ? undefined : + new Date(parentComment.publishedAt).getTime(), + author: { + name: parentComment.authorDisplayName ?? undefined, + url: parentComment.authorChannelUrl ?? undefined, + image: parentComment.authorProfileImageUrl ?? undefined + } + } : { + kind: "article", + description: video.snippet.description ?? undefined, + url: `https://www.youtube.com/watch?v=${videoId}`, + published: video.snippet.publishedAt == null ? undefined : + new Date(video.snippet.publishedAt).getTime(), + author: { + name: video.channel.title ?? undefined, + url: video.channel.customUrl ? + `https://www.youtube.com/${video.channel.customUrl}` : + `https://www.youtube.com/channel/${video.snippet.channelId}`, + image: getBestThumbnail(video.channel.thumbnails) + }, + image: getBestThumbnail(video.snippet.thumbnails), + tags: [category], + + } + }).then(() => console.log("Added reply for", url)) + .catch((err) => console.log("Failed to reply", url, err)); +}); diff --git a/syndications/indiekit.ts b/syndications/indiekit.ts new file mode 100644 index 00000000..6ad61fcf --- /dev/null +++ b/syndications/indiekit.ts @@ -0,0 +1,273 @@ +import fs from "fs"; +import open from 'open'; +import { fileTypeFromBuffer } from 'file-type'; +import type { Author, Post } from "~/types"; + +function getHash(data: Bun.BlobOrStringOrBuffer) { + const hasher = new Bun.CryptoHasher("md5"); + hasher.update(data); + return hasher.digest("hex"); +} + +function encode(value: unknown): string { + if (value instanceof Date) { + value = value.toISOString(); + } else if (value && typeof value === "object") { + const obj = value as Record; + return Object.keys(obj).map(key => { + if (Array.isArray(obj[key])) { + return obj[key].map(v => `${key}[]=${encode(v)}`).join("&"); + } + return `${key}=${encode(obj[key])}`; + }).join("&"); + } else if (typeof value !== "string") { + value = JSON.stringify(value); + } + return encodeURIComponent(value as string); +} + +let accessToken: string | undefined; +const TOKEN_PATH = "./syndications/indiekit_token.json"; +async function authenticate() { + if (fs.existsSync(TOKEN_PATH)) { + const file = await fs.promises.readFile(TOKEN_PATH); + try { + const json = JSON.parse(file.toString()); + if (json.exp && json.exp * 1000 > Date.now()) { + return json.access_token as string; + } + } catch (e) { + + } + } + + const authUrl = "https://indie.incremental.social/auth?" + encode({ + me: "https://www.thepaperpilot.org/", + client_id: "https://indie.incremental.social", + redirect_uri: "https://indie.incremental.social/session/auth", + state: Date.now(), + scope: "create media", + response_type: "code" + }); + console.log('Need to authorize app with indiekit. Opening oauth url...:', authUrl); + await open(authUrl); + console.log("Enter the code from that page here:"); + for await (const code of console as unknown as AsyncIterable) { + let res = await fetch("https://indie.incremental.social/auth/token", { + method: "POST", + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Accept': 'application/json' + }, + body: encode({ + grant_type: "authorization_code", + me: "https://www.thepaperpilot.org/", + code, + client_id: "https://indie.incremental.social", + redirect_uri: "https://indie.incremental.social/session/auth", + }) + }); + if (res.status !== 200) { + console.warn("Failed to authenticate with indiekit", res, await res.text()); + process.exit(0); + } + const token_info = await res.json() as { access_token: string; }; + res = await fetch("https://indie.incremental.social/auth/introspect", { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + "Accept": "application/json", + "Authorization": `Bearer ${token_info.access_token}` + }, + body: encode({ token: token_info.access_token }) + }); + if (res.status !== 200) { + console.warn("Failed to authenticate with indiekit", res, await res.text()); + process.exit(0); + } + await res.json().then(res => { + fs.promises.writeFile("./syndications/indiekit_token.json", + JSON.stringify({ ...res, ...token_info })); + }); + return token_info.access_token; + } +} + +let retries = 0; +async function sendPost(body: Record) { + if (accessToken == null) { + accessToken = await authenticate(); + } + + await fetch("https://indie.incremental.social/micropub", { + method: "POST", + headers: { + 'Authorization': `Bearer ${accessToken}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ type: "h-entry", properties: body }) + }).then(async res => { + if (res.status === 429) { + retries++; + if (retries > 3) { + console.error("Too many retries! Giving up."); + process.exit(0); + } + await new Promise(resolve => setTimeout(resolve, 1000)); + return await sendPost(body); + } + + retries = 0; + if (res.status === 202) { + console.log(await res.json().then(r => r.success_description as string)); + } else { + console.warn("Failed to send post to indiekit", res, await res.text()); + throw res; + } + }); +} + +const MEDIA_URLS_PATH = "./syndications/media_urls.json"; +let mediaUrls: Record | undefined = undefined; +async function uploadMedia(url: string) { + if (mediaUrls == null) { + mediaUrls = fs.existsSync(MEDIA_URLS_PATH) ? + JSON.parse(fs.readFileSync(MEDIA_URLS_PATH).toString()) as {} : {}; + } + + if (url in mediaUrls) { + return mediaUrls[url]; + } + + let res = await fetch(url); + if (res.status !== 200) { + console.log("Failed to download media", url, res, await res.text()); + return undefined; + } + + // Check if its already uploaded + const blob = await res.blob(); + const { ext } = await fileTypeFromBuffer(await blob.arrayBuffer()) ?? { ext: "jpg" }; + const hash = getHash(blob); + const predictedUrl = `https://v4y3.c12.e2-2.dev/indiekit/${ext}/${hash}.${ext}`; + if ((await fetch(predictedUrl, { method: "HEAD" }).catch(console.warn))?.status === 200) { + mediaUrls[url] = predictedUrl; + await fs.promises.writeFile(MEDIA_URLS_PATH, JSON.stringify(mediaUrls)); + return predictedUrl; + } + + if (accessToken == null) { + accessToken = await authenticate(); + } + + const filename = url.split('/').pop() ?? url; + const file = new File([blob], filename, { + type: res.headers.get("content-type") ?? undefined + }); + const formData = new FormData(); + formData.append('file', file); + res = await fetch("https://indie.incremental.social/media", { + method: "POST", + headers: { + 'Authorization': `Bearer ${accessToken}`, + }, + body: formData + }); + + if (res.status !== 201) { + console.log("Failed to upload media", url, res, await res.text()); + return undefined; + } + + const location = res.headers.get("location") ?? undefined; + mediaUrls[url] = location!; + await fs.promises.writeFile(MEDIA_URLS_PATH, JSON.stringify(mediaUrls)); + + console.log("Uploaded", url, "to", location); + return location; +} + +// All these exports exist to ensure type correctness +export async function addArticle(article: { + name?: string; + content: string; + published?: Date; + category: string | string[]; + photo?: string; + originalUrl?: string; +}) { + const { photo, ...body } = article; + const preview = photo == null ? undefined : await uploadMedia(photo); + await sendPost({ ...body, preview }); +} + +export async function addBookmark(bookmark: { + 'bookmark-of': string; + name?: string; + content?: string; + published?: Date; + category: string | string[]; + photo?: string; + author?: Partial; +}) { + const { photo, author, ...body } = bookmark; + const preview = photo == null ? undefined : await uploadMedia(photo); + if (author) { + author.image = author.image == null ? undefined : await uploadMedia(author.image); + } + await sendPost({ ...body, author, preview }); +} + +export async function addFavorite(favorite: { + 'like-of': string; + name?: string; + content?: string; + published?: Date; + category: string | string[]; + photo?: string; + author?: Partial; +}) { + const { photo, author, ...body } = favorite; + const preview = photo == null ? undefined : await uploadMedia(photo); + if (author) { + author.image = author.image == null ? undefined : await uploadMedia(author.image); + } + await sendPost({ ...body, author, preview }); +} + +export async function addReply(reply: { + 'in-reply-to': string; + content: string; + published?: Date; + category: string | string[]; + photo?: string; + originalUrl?: string; + parent: Partial +}) { + const { photo, parent, ...body } = reply; + const preview = photo == null ? undefined : await uploadMedia(photo); + if (parent.image != null) { + parent.image = await uploadMedia(parent.image); + } + if (parent.author?.image != null) { + parent.author.image = await uploadMedia(parent.author.image); + } + await sendPost({ ...body, parent, preview }); +} + +export async function addRepost(repost: { + 'repost-of': string; + name?: string; + content?: string; + published?: Date; + category: string | string[]; + photo?: string; + author?: Partial; +}) { + const { photo, author, ...body } = repost; + const preview = photo == null ? undefined : await uploadMedia(photo); + if (author) { + author.image = author.image == null ? undefined : await uploadMedia(author.image); + } + await sendPost({ ...body, author, preview }); +} diff --git a/syndications/update_youtube.ts b/syndications/update_youtube.ts new file mode 100644 index 00000000..1858f8ed --- /dev/null +++ b/syndications/update_youtube.ts @@ -0,0 +1,124 @@ +import fs from "fs"; +import type { OAuth2Client } from 'google-auth-library'; +import { addBookmark, addFavorite } from "./indiekit"; +import { authorize, getBestThumbnail, getVideos, VIDEO_CATEGORIES } from "./youtube_utils"; + +let published = Date.now(); + +authorize().then(async auth => { + await updateLikes(auth); + await updateSongs(auth); + await updateFavorites(auth); +}); + +const LIKED_VIDEOS_PATH = "./syndications/liked_videos.json"; +async function updateLikes(auth: OAuth2Client) { + const videosArr = fs.existsSync(LIKED_VIDEOS_PATH) ? + JSON.parse(fs.readFileSync(LIKED_VIDEOS_PATH).toString()) as string[] : []; + const allVideos = new Set(videosArr); + + console.log("Checking for new liked videos on YouTube..."); + // TODO switch newVideos back after initial backlog uploaded + const newVideos = Array.from(allVideos); + // Object.keys(await findNewFromPlaylist(auth, "LL", allVideos)); + if (newVideos.length > 0) { + for await (const video of (await getVideos(newVideos, auth)) ?? []) { + const url = `https://www.youtube.com/watch?v=${video.id}`; + await addBookmark({ + "bookmark-of": url, + category: VIDEO_CATEGORIES[parseInt(video.snippet.categoryId ?? "1")], + // TODO switch published date after initial backlog uploaded + // published: new Date(published++), + published: new Date(video.snippet.publishedAt ?? ""), + name: video.snippet.title ?? undefined, + content: video.snippet.description ?? undefined, + photo: getBestThumbnail(video.snippet.thumbnails), + author: { + name: video.snippet.channelTitle ?? "", + url: video.channel.customUrl ? + `https://www.youtube.com/${video.channel.customUrl}` : + `https://www.youtube.com/channel/${video.snippet.channelId}`, + image: getBestThumbnail(video.channel.thumbnails) + } + }).then(() => console.log("Added bookmark for", url)) + .catch(() => console.log("Failed to bookmark", url)); + process.exit(); + } + } + + await fs.promises.writeFile(LIKED_VIDEOS_PATH, JSON.stringify(Array.from(allVideos))); +} + +const LIKED_SONGS_PATH = "./syndications/liked_songs.json"; +async function updateSongs(auth: OAuth2Client) { + const videosArr = fs.existsSync(LIKED_SONGS_PATH) ? + JSON.parse(fs.readFileSync(LIKED_SONGS_PATH).toString()) as string[] : []; + const allVideos = new Set(videosArr); + + console.log("Checking for new liked songs on YouTube..."); + // TODO switch newVideos back after initial backlog uploaded + const newVideos = Array.from(allVideos); + // Object.keys(await findNewFromPlaylist(auth, "LM", allVideos)); + if (newVideos.length > 0) { + for await (const video of (await getVideos(newVideos, auth)) ?? []) { + const url = `https://www.youtube.com/watch?v=${video.id}`; + await addBookmark({ + "bookmark-of": url, + category: "music", + // TODO switch published date after initial backlog uploaded + // published: new Date(published++), + published: new Date(video.snippet.publishedAt ?? ""), + name: "♫ " + video.snippet.title ?? undefined, + content: video.snippet.description ?? undefined, + photo: getBestThumbnail(video.snippet.thumbnails), + author: { + name: video.snippet.channelTitle ?? "", + url: video.channel.customUrl ? + `https://www.youtube.com/${video.channel.customUrl}` : + `https://www.youtube.com/channel/${video.snippet.channelId}`, + image: getBestThumbnail(video.channel.thumbnails) + } + }).then(() => console.log("Added bookmark for", url)) + .catch(() => console.log("Failed to bookmark", url)); + } + } + + await fs.promises.writeFile(LIKED_SONGS_PATH, JSON.stringify(Array.from(allVideos))); +} + +const FAVORITE_VIDEOS_PATH = "./syndications/favorite_videos.json"; +async function updateFavorites(auth: OAuth2Client) { + const videosArr = fs.existsSync(FAVORITE_VIDEOS_PATH) ? + JSON.parse(fs.readFileSync(FAVORITE_VIDEOS_PATH).toString()) as string[] : []; + const allVideos = new Set(videosArr); + + console.log("Checking for new favorites on YouTube..."); + // TODO switch newVideos back after initial backlog uploaded + const newVideos = Array.from(allVideos); + // Object.keys(await findNewFromPlaylist(auth, "FLg1YH1wAWH7JF2-64XYio0A", allVideos)); + if (newVideos.length > 0) { + for await (const video of (await getVideos(newVideos, auth)) ?? []) { + const url = `https://www.youtube.com/watch?v=${video.id}`; + await addFavorite({ + "like-of": url, + category: VIDEO_CATEGORIES[parseInt(video.snippet.categoryId ?? "1")], + // TODO switch published date after initial backlog uploaded + // published: new Date(published++), + published: new Date(video.snippet.publishedAt ?? ""), + name: video.snippet.title ?? undefined, + content: video.snippet.description ?? undefined, + photo: getBestThumbnail(video.snippet.thumbnails), + author: { + name: video.snippet.channelTitle ?? "", + url: video.channel.customUrl ? + `https://www.youtube.com/${video.channel.customUrl}` : + `https://www.youtube.com/channel/${video.snippet.channelId}`, + image: getBestThumbnail(video.channel.thumbnails) + } + }).then(() => console.log("Added favorite for", url)) + .catch(() => console.log("Failed to favorite", url)); + } + } + + await fs.promises.writeFile(FAVORITE_VIDEOS_PATH, JSON.stringify(Array.from(allVideos))); +} diff --git a/syndications/youtube_utils.ts b/syndications/youtube_utils.ts new file mode 100644 index 00000000..764f6fd4 --- /dev/null +++ b/syndications/youtube_utils.ts @@ -0,0 +1,362 @@ +import fs from "fs"; +import { google, youtube_v3 } from 'googleapis'; +import type { OAuth2Client, Credentials } from 'google-auth-library'; +import open from 'open'; + +// Maps video categories to our tags +export const VIDEO_CATEGORIES: Record = { + 1: "entertainment", + 2: "entertainment", + 10: "music", + 15: "entertainment", + 17: "entertainment", + 18: "entertainment", + 19: "blogs", + 20: "gaming", + 21: "blogs", + 22: "blogs", + 23: "comedy", + 24: "entertainment", + 25: "news", + 26: "education", + 27: "education", + 28: "science", + 29: "politics", + 30: "movies", + 31: "television", + 32: "entertainment", + 33: "movies", + 34: "comedy", + 35: "education", + 36: "entertainment", + 37: "blogs", + 38: "entertainment", + 39: "entertainment", + 40: "entertainment", + 41: "entertainment", + 42: "entertainment", + 43: "television", + 44: "entertainment" +}; + +interface YoutubeCredentials { + installed: { + client_secret: string; + client_id: string; + redirect_uris: string[]; + } +} + +type WithRequired = { + [P in S]-?: NonNullable; + }; +type Video = youtube_v3.Schema$Video & WithRequired; +type Comment = NonNullable; +type Channel = NonNullable; + +const service = google.youtube('v3'); +const OAuth2 = google.auth.OAuth2; + +const CACHE_DIR = "./syndications/cache"; +const TOKEN_DIR = "./syndications/"; +const TOKEN_PATH = TOKEN_DIR + 'youtube_token.json'; + +const SCOPES = [ + 'https://www.googleapis.com/auth/youtube.readonly', + 'https://www.googleapis.com/auth/youtube.force-ssl' +]; + +export async function authorize() { + // Load client secrets from a local file. + const content = await fs.promises.readFile('./syndications/youtube_credentials.json') + .catch(err => { + console.log('Error loading client secret file: ' + err); + process.exit(0); + }); + + // Authorize a client with the loaded credentials, then call the YouTube API. + const credentials = JSON.parse(content.toString()) as YoutubeCredentials; + + const clientSecret = credentials.installed.client_secret; + const clientId = credentials.installed.client_id; + const redirectUrl = credentials.installed.redirect_uris[0]; + const oauth2Client = new OAuth2(clientId, clientSecret, redirectUrl); + + // Check if we have previously stored a token. + const token = await fs.promises.readFile(TOKEN_PATH); + oauth2Client.credentials = JSON.parse(token.toString()); + if (!oauth2Client.credentials.expiry_date || + oauth2Client.credentials.expiry_date < Date.now()) { + getNewToken(oauth2Client); + } + + return oauth2Client; +} + +async function getNewToken(oauth2Client: OAuth2Client) { + const authUrl = oauth2Client.generateAuthUrl({ + access_type: 'offline', + scope: SCOPES + }); + console.log('Need to authorize app with youtube. Opening oauth url...:', authUrl); + await open(authUrl); + console.log("Enter the code from that page here:"); + for await (const code of console as unknown as AsyncIterable) { + oauth2Client.getToken(code, function(err, token) { + if (err || token == null) { + console.log('Error while trying to retrieve access token', err); + throw err; + } + oauth2Client.credentials = token; + storeToken(token); + }); + break; + } +} + +function storeToken(token: Credentials) { + try { + fs.mkdirSync(TOKEN_DIR); + } catch (err) { + if (err == null || typeof err !== "object" || !("code" in err) || err?.code != 'EEXIST') { + throw err; + } + } + fs.writeFile(TOKEN_PATH, JSON.stringify(token), (err) => { + if (err) throw err; + console.log('Token stored to ' + TOKEN_PATH); + }); +} + +export async function getVideos(id: string | string[], auth: OAuth2Client) { + const allVideos: Video[] = []; + + if (typeof id === "string") { + id = [id]; + } + id = id.filter(id => { + const path = `${CACHE_DIR}/video-${id}.json`; + if (fs.existsSync(path)) { + allVideos.push(JSON.parse(fs.readFileSync(path).toString()) as Video); + return false; + } + return true; + }); + + if (id.length !== 0) { + let nextPageToken: string | undefined; + do { + const videos = await service.videos.list({ + auth, + part: ["snippet"], + id, + pageToken: nextPageToken + }).catch(err => { + if (err.status === 403) { + console.log("Failed to retrieve videos - Forbidden", id, + JSON.stringify(err.response)); + return 403; + } else if (err.status === 404) { + console.log("Failed to retrieve videos - Not Found", id, + JSON.stringify(err.response)); + return 404; + } else if (err.status === 400) { + console.log("Failed to retrieve videos - Bad Request", id, + JSON.stringify(err.response)); + process.exit(0); + } else { + console.log("Failed to retrieve videos", id, err.status, + JSON.stringify(err.response)); + return err.status as number; + } + }); + if (typeof videos === "number") { + return undefined; + } + const newVideos = videos.data.items?.filter(v => + v.snippet != null && v.id != null) ?? []; + newVideos.forEach(video => { + fs.writeFileSync(`${CACHE_DIR}/video-${video.id}.json`, JSON.stringify(video)); + }); + allVideos.push(...newVideos as Video[]); + nextPageToken = videos.data.nextPageToken ?? undefined; + } while (nextPageToken); + } + + // Add channel data to each result + let allVideosChannels: (Video & { channel: Channel })[] = []; + id = Array.from(allVideos.reduce((acc, curr) => { + const path = `${CACHE_DIR}/channel-${curr.snippet.channelId}.json`; + if (fs.existsSync(path)) { + const channel = JSON.parse(fs.readFileSync(path).toString()) as Channel; + allVideosChannels.push({ ...curr, channel }); + } else { + acc.add(curr.id); + } + return acc; + }, new Set())); + + if (id.length !== 0) { + let nextPageToken: string | undefined; + do { + const channels = await service.channels.list({ + auth, + part: ["snippet"], + id, + pageToken: nextPageToken + }).catch(err => { + if (err.status === 403) { + console.log("Failed to retrieve channels - Forbidden", id, + JSON.stringify(err.response)); + return 403; + } else if (err.status === 404) { + console.log("Failed to retrieve channels - Not Found", id, + JSON.stringify(err.response)); + return 404; + } else if (err.status === 400) { + console.log("Failed to retrieve channels - Bad Request", id, + JSON.stringify(err.response)); + process.exit(0); + } else { + console.log("Failed to retrieve channels", id, err.status, + JSON.stringify(err.response)); + return err.status as number; + } + }); + if (typeof channels === "number") { + return undefined; + } + channels.data.items?.forEach(channel => { + if (channel.snippet != null && channel.id != null) { + fs.writeFileSync(`${CACHE_DIR}/channel-${channel.id}.json`, + JSON.stringify(channel.snippet)); + allVideos.filter(v => v.snippet.channelId === channel.id).forEach(video => { + allVideosChannels.push({ ...video, channel: channel.snippet as Channel }); + }); + } + }); + nextPageToken = channels.data.nextPageToken ?? undefined; + } while (nextPageToken); + } + + return allVideosChannels; +} + +export async function getComment(id: string | string[], auth: OAuth2Client) { + const allComments: Comment[] = []; + + if (typeof id === "string") { + id = [id]; + } + id = id.filter(id => { + const path = `${CACHE_DIR}/comment-${id}.json`; + if (fs.existsSync(path)) { + allComments.push(JSON.parse(fs.readFileSync(path).toString()) as Comment); + return false; + } + return true; + }); + + if (id.length !== 0) { + let nextPageToken: string | undefined; + do { + const comments = await service.comments.list({ + auth, + part: ["snippet"], + id, + pageToken: nextPageToken + }).catch(err => { + if (err.status === 403) { + console.log("Failed to retrieve comments - Forbidden", id, + JSON.stringify(err)); + return 403; + } else if (err.status === 404) { + console.log("Failed to retrieve comments - Not Found", id, + JSON.stringify(err)); + return 404; + } else if (err.status === 400) { + console.log("Failed to retrieve comments - Bad Request", id, + JSON.stringify(err)); + process.exit(0); + } else { + console.log("Failed to retrieve comments", id, err.status, + JSON.stringify(err)); + return err.status as number; + } + }); + if (typeof comments === "number") { + return undefined; + } + comments.data.items?.forEach(comment => { + if (comment.snippet != null && comment.id != null) { + fs.writeFileSync(`${CACHE_DIR}/comment-${comment.id}.json`, + JSON.stringify(comment.snippet)); + allComments.push(comment.snippet); + } + }); + nextPageToken = comments.data.nextPageToken ?? undefined; + } while (nextPageToken); + } + + return allComments; +} + +export function getBestThumbnail(thumbnails?: youtube_v3.Schema$ThumbnailDetails) { + return thumbnails?.high?.url ?? + thumbnails?.maxres?.url ?? + thumbnails?.standard?.url ?? + thumbnails?.medium?.url ?? + thumbnails?.default?.url ?? undefined; +} + +export async function findNewFromPlaylist(auth: OAuth2Client, playlistId: string, videos: Set) { + let nextPageToken: string | undefined; + const newVideos: Record = {}; + do { + const playlistItems = await service.playlistItems.list({ + auth, + playlistId, + part: ["snippet"], + maxResults: 50, + pageToken: nextPageToken + }).catch(err => { + if (err.status === 403) { + console.log("Failed to retrieve videos - Forbidden", JSON.stringify(err.response)); + return 403; + } else if (err.status === 404) { + console.log("Failed to retrieve videos - Not Found", JSON.stringify(err.response)); + return 404; + } else if (err.status === 400) { + console.log("Failed to retrieve videos - Bad Request", + JSON.stringify(err.response)); + process.exit(0); + } else { + console.log("Failed to retrieve videos", err.status, JSON.stringify(err.response)); + return err.status as number; + } + }); + if (typeof playlistItems === "number") { + break; + } + let hasInsertedNone = true; + playlistItems.data.items?.forEach(video => { + const videoId = video.snippet?.resourceId?.videoId; + if (videoId && !videos.has(videoId)) { + const url = `https://www.youtube.com/watch?v=${videoId}`; + console.log("Found new video", url); + videos.add(videoId); + newVideos[videoId] = video; + hasInsertedNone = false; + return; + } + }); + + // Exit if a page had all duplicates + if (hasInsertedNone) { + break; + } + nextPageToken = playlistItems.data.nextPageToken ?? undefined; + } while (nextPageToken); + + return newVideos; +} diff --git a/tsconfig.json b/tsconfig.json index 0f0a131b..8995d009 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,6 +10,9 @@ "lib": [ "es2021", "dom" + ], + "types": [ + "bun" ] - } + }, } diff --git a/types.d.ts b/types.d.ts index bafb2a56..1c63088b 100644 --- a/types.d.ts +++ b/types.d.ts @@ -7,7 +7,7 @@ export interface Post extends ParsedContent { description?: string; url?: string; published: number; - author?: Author; + author?: Partial; image?: string; imageAlt?: string; tags: string[];