Implement PESOS for youtube

.gitignore

@@ -12,4 +12,11 @@ assets/favorites.json
 assets/tags.json
 public/changelog
 public/garden
-public/licenses.txt
+public/licenses.txt
+
+syndications/cache
+syndications/indiekit_token.json
+syndications/liked_songs.json
+syndications/media_urls.json
+syndications/youtube_credentials.json
+syndications/youtube_token.json

nuxt.config.ts

@@ -152,6 +152,10 @@ export default async () => {
                     },
                     { rel: 'token_endpoint', href: 'https://indie.incremental.social/auth/token' },
                     { rel: 'micropub', href: 'https://indie.incremental.social/micropub' },
+                    {
+                        rel: 'indieauth-metadata',
+                        href: 'https://indie.incremental.social/.well-known/oauth-authorization-server'
+                    },
 
                     /** Feeds */
                     { ...feedProps, href: '/posts', title: 'All posts' },

package.json

@@ -11,7 +11,8 @@
     "serve": "bun run ./.output/server/index.mjs",
     "logseq-export": "run-script-os",
     "logseq-export:win32": ".\\logseq-export\\logseq-export.exe --logseqFolder ./Garden --outputFolder ./garden-output",
-    "logseq-export:linux": "chmod +x logseq-export/logseq-export && logseq-export/logseq-export --logseqFolder ./Garden --outputFolder ./garden-output"
+    "logseq-export:linux": "chmod +x logseq-export/logseq-export && logseq-export/logseq-export --logseqFolder ./Garden --outputFolder ./garden-output",
+    "update:youtube": "bun run syndications/update_youtube.ts"
   },
   "dependencies": {
     "@iconify-json/cib": "^1.2.0",
@@ -28,8 +29,13 @@
     "sass-embedded": "^1.79.2"
   },
   "devDependencies": {
+    "@types/bun": "^1.1.10",
+    "dotenv": "^16.4.5",
     "feed": "^4.2.2",
+    "file-type": "^19.5.0",
     "generate-license-file": "^3.5.1",
+    "googleapis": "^144.0.0",
+    "open": "^10.1.0",
     "run-script-os": "^1.1.6",
     "word-counting": "^1.1.4"
   }

syndications/add_youtube_reply.ts

@@ -0,0 +1,79 @@
+import { addReply } from "./indiekit";
+import { authorize, getBestThumbnail, getComment, getVideos, VIDEO_CATEGORIES } from "./youtube_utils";
+
+let commentId: string, videoId: string;
+
+async function run() {
+    console.log("Enter the reply to add here:");
+    for await (const url of console as unknown as AsyncIterable<string>) {
+        const [_, video, comment] = url.match(
+            /https?:\/\/(?:www\.)?youtube.com\/watch\?v=([0-9a-zA-Z_\.-]+)&lc=([0-9a-zA-Z_\.-]+)/
+        ) as string[];
+        
+        if (!video || !comment) {
+            console.error("Could not parse video and comment id from url", url);
+        } else {
+            commentId = comment;
+            videoId = video;
+            break;
+        }
+    }
+}
+
+run()
+.then(authorize)
+.then(async auth => {
+    const url = `https://www.youtube.com/watch?v=${videoId}&lc=${commentId}`;
+    const comment = (await getComment(commentId, auth))?.[0];
+    const video = (await getVideos(videoId, auth))?.[0];
+    if (!comment || !video) {
+        console.log("Couldn't retrieve either the comment or video", comment, video);
+        return;
+    }
+    
+    const parentComment = comment.parentId == null ? undefined :
+        (await getComment(comment.parentId, auth))?.[0];
+
+    const category = VIDEO_CATEGORIES[parseInt(video.snippet.categoryId ?? "1")];
+    await addReply({
+        "in-reply-to": parentComment ?
+            `https://www.youtube.com/watch?v=${videoId}&lc=${comment.parentId!}` :
+            `https://www.youtube.com/watch?v=${videoId}`,
+        content: comment.textDisplay ?? "",
+        published: comment.publishedAt == null ? undefined : new Date(comment.publishedAt),
+        category,
+        originalUrl: url,
+        parent: parentComment ? {
+            kind: "reply",
+            description: parentComment.textDisplay ?? "",
+            syndications: [
+                `https://www.youtube.com/watch?v=${videoId}&lc=${comment.parentId!}`
+            ],
+            tags: [category],
+            published: parentComment.publishedAt == null ? undefined :
+                new Date(parentComment.publishedAt).getTime(),
+            author: {
+                name: parentComment.authorDisplayName ?? undefined,
+                url: parentComment.authorChannelUrl ?? undefined,
+                image: parentComment.authorProfileImageUrl ?? undefined
+            }
+        } : {
+            kind: "article",
+            description: video.snippet.description ?? undefined,
+            url: `https://www.youtube.com/watch?v=${videoId}`,
+            published: video.snippet.publishedAt == null ? undefined :
+                new Date(video.snippet.publishedAt).getTime(),
+            author: {
+                name: video.channel.title ?? undefined,
+                url: video.channel.customUrl ?
+                    `https://www.youtube.com/${video.channel.customUrl}` : 
+                    `https://www.youtube.com/channel/${video.snippet.channelId}`,
+                image: getBestThumbnail(video.channel.thumbnails)
+            },
+            image: getBestThumbnail(video.snippet.thumbnails),
+            tags: [category],
+
+        }
+    }).then(() => console.log("Added reply for", url))
+      .catch((err) => console.log("Failed to reply", url, err));
+});

syndications/indiekit.ts

@@ -0,0 +1,273 @@
+import fs from "fs";
+import open from 'open';
+import { fileTypeFromBuffer } from 'file-type';
+import type { Author, Post } from "~/types";
+
+function getHash(data: Bun.BlobOrStringOrBuffer) {
+    const hasher = new Bun.CryptoHasher("md5");
+    hasher.update(data);
+    return hasher.digest("hex");
+}
+
+function encode(value: unknown): string {
+    if (value instanceof Date) {
+        value = value.toISOString();
+    } else if (value && typeof value === "object") {
+        const obj = value as Record<string, unknown>;
+        return Object.keys(obj).map(key => {
+            if (Array.isArray(obj[key])) {
+                return obj[key].map(v => `${key}[]=${encode(v)}`).join("&");
+            }
+            return `${key}=${encode(obj[key])}`;
+        }).join("&");
+    } else if (typeof value !== "string") {
+        value = JSON.stringify(value);
+    }
+    return encodeURIComponent(value as string);
+}
+
+let accessToken: string | undefined;
+const TOKEN_PATH = "./syndications/indiekit_token.json";
+async function authenticate() {
+    if (fs.existsSync(TOKEN_PATH)) {
+        const file = await fs.promises.readFile(TOKEN_PATH);
+        try {
+            const json = JSON.parse(file.toString());
+            if (json.exp && json.exp * 1000 > Date.now()) {
+                return json.access_token as string;
+            }
+        } catch (e) {
+
+        }
+    }
+
+    const authUrl = "https://indie.incremental.social/auth?" + encode({
+        me: "https://www.thepaperpilot.org/",
+        client_id: "https://indie.incremental.social",
+        redirect_uri: "https://indie.incremental.social/session/auth",
+        state: Date.now(),
+        scope: "create media",
+        response_type: "code"
+    });
+    console.log('Need to authorize app with indiekit. Opening oauth url...:', authUrl);
+    await open(authUrl);
+    console.log("Enter the code from that page here:");
+    for await (const code of console as unknown as AsyncIterable<string>) {
+        let res = await fetch("https://indie.incremental.social/auth/token", {
+            method: "POST",
+            headers: {
+                'Content-Type': 'application/x-www-form-urlencoded',
+                'Accept': 'application/json'
+            },
+            body: encode({
+                grant_type: "authorization_code",
+                me: "https://www.thepaperpilot.org/",
+                code,
+                client_id: "https://indie.incremental.social",
+                redirect_uri: "https://indie.incremental.social/session/auth",
+            })
+        });
+        if (res.status !== 200) {
+            console.warn("Failed to authenticate with indiekit", res, await res.text());
+            process.exit(0);
+        }
+        const token_info = await res.json() as { access_token: string; };
+        res = await fetch("https://indie.incremental.social/auth/introspect", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/x-www-form-urlencoded",
+                "Accept": "application/json",
+                "Authorization": `Bearer ${token_info.access_token}`
+            },
+            body: encode({ token: token_info.access_token })
+        });
+        if (res.status !== 200) {
+            console.warn("Failed to authenticate with indiekit", res, await res.text());
+            process.exit(0);
+        }
+        await res.json().then(res => {
+            fs.promises.writeFile("./syndications/indiekit_token.json",
+                JSON.stringify({ ...res, ...token_info }));
+        });
+        return token_info.access_token;
+    }
+}
+
+let retries = 0;
+async function sendPost(body: Record<string, unknown>) {
+    if (accessToken == null) {
+        accessToken = await authenticate();
+    }
+
+    await fetch("https://indie.incremental.social/micropub", {
+        method: "POST",
+        headers: {
+            'Authorization': `Bearer ${accessToken}`,
+            'Content-Type': 'application/json'
+        },
+        body: JSON.stringify({ type: "h-entry", properties: body })
+    }).then(async res => {
+        if (res.status === 429) {
+            retries++;
+            if (retries > 3) {
+                console.error("Too many retries! Giving up.");
+                process.exit(0);
+            }
+            await new Promise(resolve => setTimeout(resolve, 1000));
+            return await sendPost(body);
+        }
+
+        retries = 0;
+        if (res.status === 202) {
+            console.log(await res.json().then(r => r.success_description as string));
+        } else {
+            console.warn("Failed to send post to indiekit", res, await res.text());
+            throw res;
+        }
+    });
+}
+
+const MEDIA_URLS_PATH = "./syndications/media_urls.json";
+let mediaUrls: Record<string, string> | undefined = undefined;
+async function uploadMedia(url: string) {
+    if (mediaUrls == null) {
+        mediaUrls = fs.existsSync(MEDIA_URLS_PATH) ?
+            JSON.parse(fs.readFileSync(MEDIA_URLS_PATH).toString()) as {} : {};
+    }
+
+    if (url in mediaUrls) {
+        return mediaUrls[url];
+    }
+
+    let res = await fetch(url);
+    if (res.status !== 200) {
+        console.log("Failed to download media", url, res, await res.text());
+        return undefined;
+    }
+
+    // Check if its already uploaded
+    const blob = await res.blob();
+    const { ext } = await fileTypeFromBuffer(await blob.arrayBuffer()) ?? { ext: "jpg" };
+    const hash = getHash(blob);
+    const predictedUrl = `https://v4y3.c12.e2-2.dev/indiekit/${ext}/${hash}.${ext}`;
+    if ((await fetch(predictedUrl, { method: "HEAD" }).catch(console.warn))?.status === 200) {
+        mediaUrls[url] = predictedUrl;
+        await fs.promises.writeFile(MEDIA_URLS_PATH, JSON.stringify(mediaUrls));
+        return predictedUrl;
+    }
+
+    if (accessToken == null) {
+        accessToken = await authenticate();
+    }
+
+    const filename = url.split('/').pop() ?? url;
+    const file = new File([blob], filename, {
+        type: res.headers.get("content-type") ?? undefined
+    });
+    const formData = new FormData();
+    formData.append('file', file);
+    res = await fetch("https://indie.incremental.social/media", {
+        method: "POST",
+        headers: {
+            'Authorization': `Bearer ${accessToken}`,
+        },
+        body: formData
+    });
+    
+    if (res.status !== 201) {
+        console.log("Failed to upload media", url, res, await res.text());
+        return undefined;
+    }
+
+    const location = res.headers.get("location") ?? undefined;
+    mediaUrls[url] = location!;
+    await fs.promises.writeFile(MEDIA_URLS_PATH, JSON.stringify(mediaUrls));
+
+    console.log("Uploaded", url, "to", location);
+    return location;
+}
+
+// All these exports exist to ensure type correctness
+export async function addArticle(article: {
+    name?: string;
+    content: string;
+    published?: Date;
+    category: string | string[];
+    photo?: string;
+    originalUrl?: string;
+}) {
+    const { photo, ...body } = article;
+    const preview = photo == null ? undefined : await uploadMedia(photo);
+    await sendPost({ ...body, preview });
+}
+
+export async function addBookmark(bookmark: {
+    'bookmark-of': string;
+    name?: string;
+    content?: string;
+    published?: Date;
+    category: string | string[];
+    photo?: string;
+    author?: Partial<Author>;
+}) {
+    const { photo, author, ...body } = bookmark;
+    const preview = photo == null ? undefined : await uploadMedia(photo);
+    if (author) {
+        author.image = author.image == null ? undefined : await uploadMedia(author.image);
+    }
+    await sendPost({ ...body, author, preview });
+}
+
+export async function addFavorite(favorite: {
+    'like-of': string;
+    name?: string;
+    content?: string;
+    published?: Date;
+    category: string | string[];
+    photo?: string;
+    author?: Partial<Author>;
+}) {
+    const { photo, author, ...body } = favorite;
+    const preview = photo == null ? undefined : await uploadMedia(photo);
+    if (author) {
+        author.image = author.image == null ? undefined : await uploadMedia(author.image);
+    }
+    await sendPost({ ...body, author, preview });
+}
+
+export async function addReply(reply: {
+    'in-reply-to': string;
+    content: string;
+    published?: Date;
+    category: string | string[];
+    photo?: string;
+    originalUrl?: string;
+    parent: Partial<Post>
+}) {
+    const { photo, parent, ...body } = reply;
+    const preview = photo == null ? undefined : await uploadMedia(photo);
+    if (parent.image != null) {
+        parent.image = await uploadMedia(parent.image);
+    }
+    if (parent.author?.image != null) {
+        parent.author.image = await uploadMedia(parent.author.image);
+    }
+    await sendPost({ ...body, parent, preview });
+}
+
+export async function addRepost(repost: {
+    'repost-of': string;
+    name?: string;
+    content?: string;
+    published?: Date;
+    category: string | string[];
+    photo?: string;
+    author?: Partial<Author>;
+}) {
+    const { photo, author, ...body } = repost;
+    const preview = photo == null ? undefined : await uploadMedia(photo);
+    if (author) {
+        author.image = author.image == null ? undefined : await uploadMedia(author.image);
+    }
+    await sendPost({ ...body, author, preview });
+}

syndications/update_youtube.ts

@@ -0,0 +1,124 @@
+import fs from "fs";
+import type { OAuth2Client } from 'google-auth-library';
+import { addBookmark, addFavorite } from "./indiekit";
+import { authorize, getBestThumbnail, getVideos, VIDEO_CATEGORIES } from "./youtube_utils";
+
+let published = Date.now();
+
+authorize().then(async auth => {
+    await updateLikes(auth);
+    await updateSongs(auth);
+    await updateFavorites(auth);
+});
+
+const LIKED_VIDEOS_PATH = "./syndications/liked_videos.json";
+async function updateLikes(auth: OAuth2Client) {
+    const videosArr = fs.existsSync(LIKED_VIDEOS_PATH) ?
+        JSON.parse(fs.readFileSync(LIKED_VIDEOS_PATH).toString()) as string[] : [];
+    const allVideos = new Set(videosArr);
+
+    console.log("Checking for new liked videos on YouTube...");
+    // TODO switch newVideos back after initial backlog uploaded
+    const newVideos = Array.from(allVideos);
+        // Object.keys(await findNewFromPlaylist(auth, "LL", allVideos));
+    if (newVideos.length > 0) {
+        for await (const video of (await getVideos(newVideos, auth)) ?? []) {
+            const url = `https://www.youtube.com/watch?v=${video.id}`;
+            await addBookmark({
+                "bookmark-of": url,
+                category: VIDEO_CATEGORIES[parseInt(video.snippet.categoryId ?? "1")],
+                // TODO switch published date after initial backlog uploaded
+                // published: new Date(published++),
+                published: new Date(video.snippet.publishedAt ?? ""),
+                name: video.snippet.title ?? undefined,
+                content: video.snippet.description ?? undefined,
+                photo: getBestThumbnail(video.snippet.thumbnails),
+                author: {
+                    name: video.snippet.channelTitle ?? "",
+                    url: video.channel.customUrl ?
+                        `https://www.youtube.com/${video.channel.customUrl}` :
+                        `https://www.youtube.com/channel/${video.snippet.channelId}`,
+                    image: getBestThumbnail(video.channel.thumbnails)
+                }
+            }).then(() => console.log("Added bookmark for", url))
+              .catch(() => console.log("Failed to bookmark", url));
+              process.exit();
+        }
+    }
+
+    await fs.promises.writeFile(LIKED_VIDEOS_PATH, JSON.stringify(Array.from(allVideos)));
+}
+
+const LIKED_SONGS_PATH = "./syndications/liked_songs.json";
+async function updateSongs(auth: OAuth2Client) {
+    const videosArr = fs.existsSync(LIKED_SONGS_PATH) ?
+        JSON.parse(fs.readFileSync(LIKED_SONGS_PATH).toString()) as string[] : [];
+    const allVideos = new Set(videosArr);
+
+    console.log("Checking for new liked songs on YouTube...");
+    // TODO switch newVideos back after initial backlog uploaded
+    const newVideos = Array.from(allVideos);
+        // Object.keys(await findNewFromPlaylist(auth, "LM", allVideos));
+    if (newVideos.length > 0) {
+        for await (const video of (await getVideos(newVideos, auth)) ?? []) {
+            const url = `https://www.youtube.com/watch?v=${video.id}`;
+            await addBookmark({
+                "bookmark-of": url,
+                category: "music",
+                // TODO switch published date after initial backlog uploaded
+                // published: new Date(published++),
+                published: new Date(video.snippet.publishedAt ?? ""),
+                name: "♫ " + video.snippet.title ?? undefined,
+                content: video.snippet.description ?? undefined,
+                photo: getBestThumbnail(video.snippet.thumbnails),
+                author: {
+                    name: video.snippet.channelTitle ?? "",
+                    url: video.channel.customUrl ?
+                        `https://www.youtube.com/${video.channel.customUrl}` :
+                        `https://www.youtube.com/channel/${video.snippet.channelId}`,
+                    image: getBestThumbnail(video.channel.thumbnails)
+                }
+            }).then(() => console.log("Added bookmark for", url))
+              .catch(() => console.log("Failed to bookmark", url));
+        }
+    }
+
+    await fs.promises.writeFile(LIKED_SONGS_PATH, JSON.stringify(Array.from(allVideos)));
+}
+
+const FAVORITE_VIDEOS_PATH = "./syndications/favorite_videos.json";
+async function updateFavorites(auth: OAuth2Client) {
+    const videosArr = fs.existsSync(FAVORITE_VIDEOS_PATH) ?
+        JSON.parse(fs.readFileSync(FAVORITE_VIDEOS_PATH).toString()) as string[] : [];
+    const allVideos = new Set(videosArr);
+
+    console.log("Checking for new favorites on YouTube...");
+    // TODO switch newVideos back after initial backlog uploaded
+    const newVideos = Array.from(allVideos);
+        // Object.keys(await findNewFromPlaylist(auth, "FLg1YH1wAWH7JF2-64XYio0A", allVideos));
+    if (newVideos.length > 0) {
+        for await (const video of (await getVideos(newVideos, auth)) ?? []) {
+            const url = `https://www.youtube.com/watch?v=${video.id}`;
+            await addFavorite({
+                "like-of": url,
+                category: VIDEO_CATEGORIES[parseInt(video.snippet.categoryId ?? "1")],
+                // TODO switch published date after initial backlog uploaded
+                // published: new Date(published++),
+                published: new Date(video.snippet.publishedAt ?? ""),
+                name: video.snippet.title ?? undefined,
+                content: video.snippet.description ?? undefined,
+                photo: getBestThumbnail(video.snippet.thumbnails),
+                author: {
+                    name: video.snippet.channelTitle ?? "",
+                    url: video.channel.customUrl ?
+                        `https://www.youtube.com/${video.channel.customUrl}` :
+                        `https://www.youtube.com/channel/${video.snippet.channelId}`,
+                    image: getBestThumbnail(video.channel.thumbnails)
+                }
+            }).then(() => console.log("Added favorite for", url))
+              .catch(() => console.log("Failed to favorite", url));
+        }
+    }
+
+    await fs.promises.writeFile(FAVORITE_VIDEOS_PATH, JSON.stringify(Array.from(allVideos)));
+}

syndications/youtube_utils.ts

@@ -0,0 +1,362 @@
+import fs from "fs";
+import { google, youtube_v3 } from 'googleapis';
+import type { OAuth2Client, Credentials } from 'google-auth-library';
+import open from 'open';
+
+// Maps video categories to our tags
+export const VIDEO_CATEGORIES: Record<number, string> = {
+    1: "entertainment",
+    2: "entertainment",
+    10: "music",
+    15: "entertainment",
+    17: "entertainment",
+    18: "entertainment",
+    19: "blogs",
+    20: "gaming",
+    21: "blogs",
+    22: "blogs",
+    23: "comedy",
+    24: "entertainment",
+    25: "news",
+    26: "education",
+    27: "education",
+    28: "science",
+    29: "politics",
+    30: "movies",
+    31: "television",
+    32: "entertainment",
+    33: "movies",
+    34: "comedy",
+    35: "education",
+    36: "entertainment",
+    37: "blogs",
+    38: "entertainment",
+    39: "entertainment",
+    40: "entertainment",
+    41: "entertainment",
+    42: "entertainment",
+    43: "television",
+    44: "entertainment"
+};
+
+interface YoutubeCredentials {
+    installed: {
+        client_secret: string;
+        client_id: string;
+        redirect_uris: string[];
+    }
+}
+
+type WithRequired<T, S extends keyof T> = {
+    [P in S]-?: NonNullable<T[P]>;
+  };
+type Video = youtube_v3.Schema$Video & WithRequired<youtube_v3.Schema$Video, "snippet" | "id">;
+type Comment = NonNullable<youtube_v3.Schema$Comment["snippet"]>;
+type Channel = NonNullable<youtube_v3.Schema$Channel["snippet"]>;
+
+const service = google.youtube('v3');
+const OAuth2 = google.auth.OAuth2;
+
+const CACHE_DIR = "./syndications/cache";
+const TOKEN_DIR = "./syndications/";
+const TOKEN_PATH = TOKEN_DIR + 'youtube_token.json';
+
+const SCOPES = [
+    'https://www.googleapis.com/auth/youtube.readonly',
+    'https://www.googleapis.com/auth/youtube.force-ssl'
+];
+
+export async function authorize() {
+    // Load client secrets from a local file.
+    const content = await fs.promises.readFile('./syndications/youtube_credentials.json')
+        .catch(err => {
+            console.log('Error loading client secret file: ' + err);
+            process.exit(0);
+        });
+
+    // Authorize a client with the loaded credentials, then call the YouTube API.
+    const credentials = JSON.parse(content.toString()) as YoutubeCredentials;
+
+    const clientSecret = credentials.installed.client_secret;
+    const clientId = credentials.installed.client_id;
+    const redirectUrl = credentials.installed.redirect_uris[0];
+    const oauth2Client = new OAuth2(clientId, clientSecret, redirectUrl);
+    
+    // Check if we have previously stored a token.
+    const token = await fs.promises.readFile(TOKEN_PATH);
+    oauth2Client.credentials = JSON.parse(token.toString());
+    if (!oauth2Client.credentials.expiry_date ||
+        oauth2Client.credentials.expiry_date < Date.now()) {
+        getNewToken(oauth2Client);
+    }
+
+    return oauth2Client;
+}
+
+async function getNewToken(oauth2Client: OAuth2Client) {
+    const authUrl = oauth2Client.generateAuthUrl({
+        access_type: 'offline',
+        scope: SCOPES
+    });
+    console.log('Need to authorize app with youtube. Opening oauth url...:', authUrl);
+    await open(authUrl);
+    console.log("Enter the code from that page here:");
+    for await (const code of console as unknown as AsyncIterable<string>) {
+        oauth2Client.getToken(code, function(err, token) {
+            if (err || token == null) {
+                console.log('Error while trying to retrieve access token', err);
+                throw err;
+            }
+            oauth2Client.credentials = token;
+            storeToken(token);
+        });
+        break;
+    }
+}
+
+function storeToken(token: Credentials) {
+    try {
+        fs.mkdirSync(TOKEN_DIR);
+    } catch (err) {
+        if (err == null || typeof err !== "object" || !("code" in err) || err?.code != 'EEXIST') {
+            throw err;
+        }
+    }
+    fs.writeFile(TOKEN_PATH, JSON.stringify(token), (err) => {
+        if (err) throw err;
+        console.log('Token stored to ' + TOKEN_PATH);
+    });
+}
+
+export async function getVideos(id: string | string[], auth: OAuth2Client) {
+    const allVideos: Video[] = [];
+
+    if (typeof id === "string") {
+        id = [id];
+    }
+    id = id.filter(id => {
+        const path = `${CACHE_DIR}/video-${id}.json`;
+        if (fs.existsSync(path)) {
+            allVideos.push(JSON.parse(fs.readFileSync(path).toString()) as Video);
+            return false;
+        }
+        return true;
+    });
+
+    if (id.length !== 0) {
+        let nextPageToken: string | undefined;
+        do {
+            const videos = await service.videos.list({
+                auth,
+                part: ["snippet"],
+                id,
+                pageToken: nextPageToken
+            }).catch(err => {
+                if (err.status === 403) {
+                    console.log("Failed to retrieve videos - Forbidden", id,
+                        JSON.stringify(err.response));
+                    return 403;
+                } else if (err.status === 404) {
+                    console.log("Failed to retrieve videos - Not Found", id,
+                        JSON.stringify(err.response));
+                    return 404;
+                } else if (err.status === 400) {
+                    console.log("Failed to retrieve videos - Bad Request", id,
+                        JSON.stringify(err.response));
+                    process.exit(0);
+                } else {
+                    console.log("Failed to retrieve videos", id, err.status,
+                        JSON.stringify(err.response));
+                    return err.status as number;
+                }
+            });
+            if (typeof videos === "number") {
+                return undefined;
+            }
+            const newVideos = videos.data.items?.filter(v =>
+                v.snippet != null && v.id != null) ?? [];
+            newVideos.forEach(video => {
+                fs.writeFileSync(`${CACHE_DIR}/video-${video.id}.json`, JSON.stringify(video));
+            });
+            allVideos.push(...newVideos as Video[]);
+            nextPageToken = videos.data.nextPageToken ?? undefined;
+        } while (nextPageToken);
+    }
+
+    // Add channel data to each result
+    let allVideosChannels: (Video & { channel: Channel })[] = [];
+    id = Array.from(allVideos.reduce((acc, curr) => {
+        const path = `${CACHE_DIR}/channel-${curr.snippet.channelId}.json`;
+        if (fs.existsSync(path)) {
+            const channel = JSON.parse(fs.readFileSync(path).toString()) as Channel;
+            allVideosChannels.push({ ...curr, channel });
+        } else {
+            acc.add(curr.id);
+        }
+        return acc;
+    }, new Set<string>()));
+
+    if (id.length !== 0) {
+        let nextPageToken: string | undefined;
+        do {
+            const channels = await service.channels.list({
+                auth,
+                part: ["snippet"],
+                id,
+                pageToken: nextPageToken
+            }).catch(err => {
+                if (err.status === 403) {
+                    console.log("Failed to retrieve channels - Forbidden", id,
+                        JSON.stringify(err.response));
+                    return 403;
+                } else if (err.status === 404) {
+                    console.log("Failed to retrieve channels - Not Found", id,
+                        JSON.stringify(err.response));
+                    return 404;
+                } else if (err.status === 400) {
+                    console.log("Failed to retrieve channels - Bad Request", id,
+                        JSON.stringify(err.response));
+                    process.exit(0);
+                } else {
+                    console.log("Failed to retrieve channels", id, err.status,
+                        JSON.stringify(err.response));
+                    return err.status as number;
+                }
+            });
+            if (typeof channels === "number") {
+                return undefined;
+            }
+            channels.data.items?.forEach(channel => {
+                if (channel.snippet != null && channel.id != null) {
+                    fs.writeFileSync(`${CACHE_DIR}/channel-${channel.id}.json`,
+                        JSON.stringify(channel.snippet));
+                    allVideos.filter(v => v.snippet.channelId === channel.id).forEach(video => {
+                        allVideosChannels.push({ ...video, channel: channel.snippet as Channel });
+                    });
+                }
+            });
+            nextPageToken = channels.data.nextPageToken ?? undefined;
+        } while (nextPageToken);
+    }
+
+    return allVideosChannels;
+}
+
+export async function getComment(id: string | string[], auth: OAuth2Client) {
+    const allComments: Comment[] = [];
+
+    if (typeof id === "string") {
+        id = [id];
+    }
+    id = id.filter(id => {
+        const path = `${CACHE_DIR}/comment-${id}.json`;
+        if (fs.existsSync(path)) {
+            allComments.push(JSON.parse(fs.readFileSync(path).toString()) as Comment);
+            return false;
+        }
+        return true;
+    });
+
+    if (id.length !== 0) {
+        let nextPageToken: string | undefined;
+        do {
+            const comments = await service.comments.list({
+                auth,
+                part: ["snippet"],
+                id,
+                pageToken: nextPageToken
+            }).catch(err => {
+                if (err.status === 403) {
+                    console.log("Failed to retrieve comments - Forbidden", id,
+                        JSON.stringify(err));
+                    return 403;
+                } else if (err.status === 404) {
+                    console.log("Failed to retrieve comments - Not Found", id,
+                        JSON.stringify(err));
+                    return 404;
+                } else if (err.status === 400) {
+                    console.log("Failed to retrieve comments - Bad Request", id,
+                        JSON.stringify(err));
+                    process.exit(0);
+                } else {
+                    console.log("Failed to retrieve comments", id, err.status,
+                        JSON.stringify(err));
+                    return err.status as number;
+                }
+            });
+            if (typeof comments === "number") {
+                return undefined;
+            }
+            comments.data.items?.forEach(comment => {
+                if (comment.snippet != null && comment.id != null) {
+                    fs.writeFileSync(`${CACHE_DIR}/comment-${comment.id}.json`,
+                        JSON.stringify(comment.snippet));
+                    allComments.push(comment.snippet);
+                }
+            });
+            nextPageToken = comments.data.nextPageToken ?? undefined;
+        } while (nextPageToken);
+    }
+
+    return allComments;
+}
+
+export function getBestThumbnail(thumbnails?: youtube_v3.Schema$ThumbnailDetails) {
+    return thumbnails?.high?.url ??
+        thumbnails?.maxres?.url ??
+        thumbnails?.standard?.url ??
+        thumbnails?.medium?.url ??
+        thumbnails?.default?.url ?? undefined;
+}
+
+export async function findNewFromPlaylist(auth: OAuth2Client, playlistId: string, videos: Set<string>) {
+    let nextPageToken: string | undefined;
+    const newVideos: Record<string, youtube_v3.Schema$PlaylistItem> = {};
+    do {
+        const playlistItems = await service.playlistItems.list({
+            auth,
+            playlistId,
+            part: ["snippet"],
+            maxResults: 50,
+            pageToken: nextPageToken
+        }).catch(err => {
+            if (err.status === 403) {
+                console.log("Failed to retrieve videos - Forbidden", JSON.stringify(err.response));
+                return 403;
+            } else if (err.status === 404) {
+                console.log("Failed to retrieve videos - Not Found", JSON.stringify(err.response));
+                return 404;
+            } else if (err.status === 400) {
+                console.log("Failed to retrieve videos - Bad Request",
+                    JSON.stringify(err.response));
+                process.exit(0);
+            } else {
+                console.log("Failed to retrieve videos", err.status, JSON.stringify(err.response));
+                return err.status as number;
+            }
+        });
+        if (typeof playlistItems === "number") {
+            break;
+        }
+        let hasInsertedNone = true;
+        playlistItems.data.items?.forEach(video => {
+            const videoId = video.snippet?.resourceId?.videoId;
+            if (videoId && !videos.has(videoId)) {
+                const url = `https://www.youtube.com/watch?v=${videoId}`;
+                console.log("Found new video", url);
+                videos.add(videoId);
+                newVideos[videoId] = video;
+                hasInsertedNone = false;
+                return;
+            }
+        });
+
+        // Exit if a page had all duplicates
+        if (hasInsertedNone) {
+            break;
+        }
+        nextPageToken = playlistItems.data.nextPageToken ?? undefined;
+    } while (nextPageToken);
+
+    return newVideos;
+}

tsconfig.json

@@ -10,6 +10,9 @@
     "lib": [
       "es2021",
       "dom"
+    ],
+    "types": [
+      "bun"
     ]
-  }
+  },
 }

types.d.ts

@@ -7,7 +7,7 @@ export interface Post extends ParsedContent {
     description?: string;
     url?: string;
     published: number;
-    author?: Author;
+    author?: Partial<Author>;
     image?: string;
     imageAlt?: string;
     tags: string[];