diff --git a/.gitignore b/.gitignore index 0efae0e..53cb96e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ repos .claude deploy.yml claude.md +feed diff --git a/compose.yml b/compose.yml index 54a8957..5f51621 100644 --- a/compose.yml +++ b/compose.yml @@ -98,18 +98,6 @@ services: - ./envs/social-app command: "/usr/bin/bskyweb serve" - jetstream: - build: - context: ./repos/jetstream/ - dockerfile: Dockerfile - ports: - - 6008:6008 - volumes: - - ./data/jetstream:/data - restart: always - env_file: - - ./envs/jetstream - ozone: ports: - 2585:3000 @@ -143,3 +131,26 @@ services: depends_on: database: condition: service_healthy + + jetstream: + build: + context: ./repos/jetstream/ + dockerfile: Dockerfile + ports: + - 6008:6008 + volumes: + - ./data/jetstream:/data + restart: always + env_file: + - ./envs/jetstream + + feed: + ports: + - 2587:3000 + build: + context: ./repos/feed-generator/ + restart: always + env_file: + - ./envs/feed + volumes: + - ./data/feed:/data diff --git a/install.zsh b/install.zsh index bba3815..6879daf 100755 --- a/install.zsh +++ b/install.zsh @@ -1,7 +1,9 @@ #!/bin/zsh function at-repos-env() { + APP_PASSWORD=xxx host=syu.is + handle=ai.syui.ai did=did:plc:6qyecktefllvenje24fcxnie repos=( "https://github.com/did-method-plc/did-method-plc" @@ -20,6 +22,7 @@ function at-repos-env() { "bgs" "ozone" "social-app" + "feed" ) handles=( "syui.syui.ai" @@ -51,6 +54,7 @@ PATCH_FILES=( "140-social-app-yarn-network-timeout.patch" "130-atproto-ozone-enable-daemon-v2.patch" "190-bgs-disable-ratelimit.patch" + "200-feed-generator-custom.patch" ) function at-repos-clone() { @@ -249,6 +253,8 @@ function at-repos-patch-apply-all() { repo="atproto" elif [[ $filename == *"indigo"* || $filename == *"bgs"* ]]; then repo="indigo" + elif [[ $filename == *"feed"* ]]; then + repo="feed-generator" fi patch-apply "$title" "$repo" "$filename" @@ -406,6 +412,15 @@ function at-repos-reset-bgs-db() { done } +function at-repos-feed-generator-start-push() { + cd $d/repos/feed-generator + yarn install + FEEDGEN_HANDLE=${handle} + FEEDGEN_PASSWORD=${APP_PASSWORD} + FEEDGEN_RECORD_NAME=app + npx tsx scripts/publish.ts +} + at-repos-env case "$1" in pull) @@ -436,11 +451,15 @@ case "$1" in cd $d;docker compose down exit ;; + feed-push) + at-repos-feed-generator-start-push + exit + ;; esac case "`cat /etc/hostname`" in at) - if [ "$1" = "reset-bgs-db" ];then + if [ "$1" = "bgs-reset" ];then at-repos-reset-bgs-db exit fi diff --git a/patching/200-feed-generator-custom.patch b/patching/200-feed-generator-custom.patch new file mode 100644 index 0000000..9d7156d --- /dev/null +++ b/patching/200-feed-generator-custom.patch @@ -0,0 +1,144 @@ +diff --git a/scripts/publish.ts b/scripts/publish.ts +new file mode 100644 +index 0000000..966edcf +--- /dev/null ++++ b/scripts/publish.ts +@@ -0,0 +1,64 @@ ++import dotenv from 'dotenv' ++import { AtpAgent, BlobRef, AppBskyFeedDefs } from '@atproto/api' ++import fs from 'fs/promises' ++import { ids } from '../src/lexicon/lexicons' ++ ++const run = async () => { ++ dotenv.config() ++ ++ const handle = process.env.FEEDGEN_HANDLE ++ const password = process.env.FEEDGEN_PASSWORD ++ const recordName = process.env.FEEDGEN_RECORD_NAME || 'app' ++ const displayName = process.env.FEEDGEN_DISPLAY_NAME || 'App Feed' ++ const description = process.env.FEEDGEN_DESCRIPTION || 'Automated App Feed' ++ const avatar = process.env.FEEDGEN_AVATAR ++ const service = process.env.FEEDGEN_SERVICE_URL || 'https://syu.is' ++ ++ if (!handle || !password) { ++ throw new Error('Please provide FEEDGEN_HANDLE and FEEDGEN_PASSWORD environment variables') ++ } ++ ++ if (!process.env.FEEDGEN_SERVICE_DID && !process.env.FEEDGEN_HOSTNAME) { ++ throw new Error('Please provide a hostname in the .env file') ++ } ++ ++ const feedGenDid = ++ process.env.FEEDGEN_SERVICE_DID ?? `did:web:${process.env.FEEDGEN_HOSTNAME}` ++ ++ const agent = new AtpAgent({ service }) ++ await agent.login({ identifier: handle, password }) ++ ++ let avatarRef: BlobRef | undefined ++ if (avatar) { ++ let encoding: string ++ if (avatar.endsWith('png')) { ++ encoding = 'image/png' ++ } else if (avatar.endsWith('jpg') || avatar.endsWith('jpeg')) { ++ encoding = 'image/jpeg' ++ } else { ++ throw new Error('expected png or jpeg') ++ } ++ const img = await fs.readFile(avatar) ++ const blobRes = await agent.api.com.atproto.repo.uploadBlob(img, { ++ encoding, ++ }) ++ avatarRef = blobRes.data.blob ++ } ++ ++ await agent.api.com.atproto.repo.putRecord({ ++ repo: agent.session?.did ?? '', ++ collection: ids.AppBskyFeedGenerator, ++ rkey: recordName, ++ record: { ++ did: feedGenDid, ++ displayName: displayName, ++ description: description, ++ avatar: avatarRef, ++ createdAt: new Date().toISOString(), ++ }, ++ }) ++ ++ console.log('All done 🎉') ++} ++ ++run() +diff --git a/src/algos/app.ts b/src/algos/app.ts +new file mode 100644 +index 0000000..2376be9 +--- /dev/null ++++ b/src/algos/app.ts +@@ -0,0 +1,35 @@ ++import { QueryParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton' ++import { AppContext } from '../config' ++ ++// max 15 chars ++export const shortname = 'app' ++ ++export const handler = async (ctx: AppContext, params: QueryParams) => { ++ let builder = ctx.db ++ .selectFrom('post') ++ .selectAll() ++ .orderBy('indexedAt', 'desc') ++ .orderBy('cid', 'desc') ++ .limit(params.limit) ++ ++ if (params.cursor) { ++ const timeStr = new Date(parseInt(params.cursor, 10)).toISOString() ++ builder = builder.where('post.indexedAt', '<', timeStr) ++ } ++ const res = await builder.execute() ++ ++ const feed = res.map((row) => ({ ++ post: row.uri, ++ })) ++ ++ let cursor: string | undefined ++ const last = res.at(-1) ++ if (last) { ++ cursor = new Date(last.indexedAt).getTime().toString(10) ++ } ++ ++ return { ++ cursor, ++ feed, ++ } ++} +diff --git a/src/algos/index.ts b/src/algos/index.ts +index b7ee48a..102cb93 100644 +--- a/src/algos/index.ts ++++ b/src/algos/index.ts +@@ -4,11 +4,13 @@ import { + OutputSchema as AlgoOutput, + } from '../lexicon/types/app/bsky/feed/getFeedSkeleton' + import * as whatsAlf from './whats-alf' ++import * as app from './app' + + type AlgoHandler = (ctx: AppContext, params: QueryParams) => Promise + + const algos: Record = { + [whatsAlf.shortname]: whatsAlf.handler, ++ [app.shortname]: app.handler, + } + + export default algos +diff --git a/src/subscription.ts b/src/subscription.ts +index 0422a03..d591ef9 100644 +--- a/src/subscription.ts ++++ b/src/subscription.ts +@@ -19,10 +19,6 @@ export class FirehoseSubscription extends FirehoseSubscriptionBase { + + const postsToDelete = ops.posts.deletes.map((del) => del.uri) + const postsToCreate = ops.posts.creates +- .filter((create) => { +- // only alf-related posts +- return create.record.text.toLowerCase().includes('alf') +- }) + .map((create) => { + // map alf-related posts to a db row + return {