diff --git a/README.md b/README.md index c12b9bf..6b1798c 100644 --- a/README.md +++ b/README.md @@ -39,10 +39,29 @@ $ curl -sL "syu.is/xrpc/com.atproto.repo.listRecords?repo=${handle}&collection=a ## feed -> at://did:plc:6qyecktefllvenje24fcxnie/app.bsky.feed.generator/cmd - -- https://api.bsky.app/xrpc/app.bsky.feed.getFeed?feed=at://did:plc:4hqjfn7m6n5hno3doamuhgef/app.bsky.feed.generator/cmd -- https://feed.syu.is/xrpc/app.bsky.feed.getFeedSkeleton?feed=at://did:plc:4hqjfn7m6n5hno3doamuhgef/app.bsky.feed.generator/cmd -- https://desc.syu.is/xrpc/app.bsky.feed.getFeedSkeleton?feed=at://did:plc:6qyecktefllvenje24fcxnie/app.bsky.feed.generator/cmd +> at://did:plc:6qyecktefllvenje24fcxnie/app.bsky.feed.generator/app +- https://syu.is/profile/ai.syui.ai/feed/app +- https://feed.syu.is/xrpc/app.bsky.feed.getFeedSkeleton?feed=at://did:plc:4hqjfn7m6n5hno3doamuhgef/app.bsky.feed.generator/app +```json +{ + "uri": "at://did:plc:6qyecktefllvenje24fcxnie/app.bsky.feed.generator/app", + "cid": "bafyreifme6g5mhuiwfmjaubwnkoyvwak6c6zvcy4uv3giikxvqpvhqdtau", + "value": { + "did": "did:web:feed.syu.is", + "$type": "app.bsky.feed.generator", + "avatar": { + "$type": "blob", + "ref": { + "$link": "bafkreigo3ucp32carhbn3chfc3hlf6i7f4rplojc76iylihzpifyexi24y" + }, + "mimeType": "image/jpeg", + "size": 375259 + }, + "createdAt": "2025-12-06T09:07:32Z", + "description": "Automated App Feed", + "displayName": "App Feed" + } +} +``` diff --git a/compose.yml b/compose.yml index 54a8957..ca1a873 100644 --- a/compose.yml +++ b/compose.yml @@ -98,18 +98,6 @@ services: - ./envs/social-app command: "/usr/bin/bskyweb serve" - jetstream: - build: - context: ./repos/jetstream/ - dockerfile: Dockerfile - ports: - - 6008:6008 - volumes: - - ./data/jetstream:/data - restart: always - env_file: - - ./envs/jetstream - ozone: ports: - 2585:3000 @@ -143,3 +131,26 @@ services: depends_on: database: condition: service_healthy + + jetstream: + build: + context: ./repos/jetstream/ + dockerfile: Dockerfile + ports: + - 6008:6008 + volumes: + - ./data/jetstream:/data + restart: always + env_file: + - ./envs/jetstream + + feed: + ports: + - 2587:3000 + build: + context: ./repos/feed-generator/ + restart: always + env_file: + - ./envs/feed + volumes: + - ./data/feed:/data/ diff --git a/envs/feed b/envs/feed new file mode 100644 index 0000000..60f82c3 --- /dev/null +++ b/envs/feed @@ -0,0 +1,7 @@ +FEEDGEN_PORT=3000 +FEEDGEN_LISTENHOST=0.0.0.0 +FEEDGEN_SQLITE_LOCATION=/data/db.sqlite +FEEDGEN_HOSTNAME=feed.syu.is +FEEDGEN_PUBLISHER_DID=did:plc:6qyecktefllvenje24fcxnie +FEEDGEN_SUBSCRIPTION_ENDPOINT=ws://bgs:2470 +FEEDGEN_SERVICE_DID=did:web:feed.syu.is diff --git a/install.zsh b/install.zsh index bba3815..58cdca6 100755 --- a/install.zsh +++ b/install.zsh @@ -1,7 +1,9 @@ #!/bin/zsh function at-repos-env() { + APP_PASSWORD=xxx host=syu.is + handle=ai.syui.ai did=did:plc:6qyecktefllvenje24fcxnie repos=( "https://github.com/did-method-plc/did-method-plc" @@ -20,6 +22,7 @@ function at-repos-env() { "bgs" "ozone" "social-app" + "feed" ) handles=( "syui.syui.ai" @@ -51,6 +54,7 @@ PATCH_FILES=( "140-social-app-yarn-network-timeout.patch" "130-atproto-ozone-enable-daemon-v2.patch" "190-bgs-disable-ratelimit.patch" + "200-feed-generator-custom.patch" ) function at-repos-clone() { @@ -78,7 +82,7 @@ function at-repos-pull() { echo $repo if [ -d $d/repos/${repo##*/} ];then cd $d/repos/${repo##*/} - git stash + git stash -u if ! git pull;then rm -rf $d/repos/${repo##*/} at-repos-clone @@ -249,6 +253,8 @@ function at-repos-patch-apply-all() { repo="atproto" elif [[ $filename == *"indigo"* || $filename == *"bgs"* ]]; then repo="indigo" + elif [[ $filename == *"feed"* ]]; then + repo="feed-generator" fi patch-apply "$title" "$repo" "$filename" @@ -337,7 +343,7 @@ function at-repos-push-docker() { service=${services[$i]} docker tag at-${service}:latest localhost:${dport}/${service}:latest docker push localhost:${dport}/${service}:latest - if [ "$service" == "ozone" ]];then + if [ "$service" == "ozone" ];then docker tag at-${service}-web:latest localhost:${dport}/${service}-web:latest docker push localhost:${dport}/${service}-web:latest fi @@ -406,6 +412,66 @@ function at-repos-reset-bgs-db() { done } +function at-repos-feed-generator-start-push() { + cd $d/repos/feed-generator + yarn install + FEEDGEN_HANDLE=${handle} + FEEDGEN_PASSWORD=${APP_PASSWORD} + FEEDGEN_RECORD_NAME=app + FEEDGEN_AVATAR=$d/repos/atproto/packages/dev-env/assets/at.png + npx tsx scripts/publish.ts +} + +function at-repos-feed-generator-update() { + + resp=$(curl -sL -X POST -H "Content-Type: application/json" -d "{\"identifier\":\"$handle\",\"password\":\"${APP_PASSWORD}\"}" https://${host}/xrpc/com.atproto.server.createSession) + token=$(echo $resp | jq -r .accessJwt) + if [ -z "$token" ] || [ "$token" == "null" ]; then + echo "Login failed: $resp" + exit 1 + fi + + avatar_json="{\"\$type\":\"blob\",\"ref\":{\"\$link\":\"${img_id}\"},\"mimeType\":\"image/jpeg\",\"size\":375259}" + + # 3. Delete cmd record + #curl -sL -X POST -H "Content-Type: application/json" -H "Authorization: Bearer $token" \ + # -d "{\"repo\":\"$handle\",\"collection\":\"app.bsky.feed.generator\",\"rkey\":\"cmd\"}" \ + # https://${host}/xrpc/com.atproto.repo.deleteRecord + + # 4. Put app record + echo "Creating app record..." + now=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + # Create JSON payload + # Note: feeding avatar_json directly into jq + payload=$(jq -n \ + --arg repo "$handle" \ + --arg collection "app.bsky.feed.generator" \ + --arg rkey "app" \ + --arg did "did:web:feed.${host}" \ + --arg type "app.bsky.feed.generator" \ + --arg created "$now" \ + --arg display "App Feed" \ + --arg desc "Automated App Feed" \ + --argjson avatar "$avatar_json" \ + '{ + repo: $repo, + collection: $collection, + rkey: $rkey, + record: { + did: $did, + "$type": $type, + createdAt: $created, + displayName: $display, + description: $desc, + avatar: $avatar + } + }') +curl -sL -X POST -H "Content-Type: application/json" -H "Authorization: Bearer $token" \ + -d @- \ + https://${host}/xrpc/com.atproto.repo.putRecord +} + at-repos-env case "$1" in pull) @@ -436,11 +502,15 @@ case "$1" in cd $d;docker compose down exit ;; + feed-push) + at-repos-feed-generator-start-push + exit + ;; esac case "`cat /etc/hostname`" in at) - if [ "$1" = "reset-bgs-db" ];then + if [ "$1" = "bgs-reset" ];then at-repos-reset-bgs-db exit fi diff --git a/patching/200-feed-generator-custom.patch b/patching/200-feed-generator-custom.patch new file mode 100644 index 0000000..8c68f4c --- /dev/null +++ b/patching/200-feed-generator-custom.patch @@ -0,0 +1,194 @@ +diff --git a/.dockerignore b/.dockerignore +new file mode 100644 +index 0000000..3c3629e +--- /dev/null ++++ b/.dockerignore +@@ -0,0 +1 @@ ++node_modules +diff --git a/Dockerfile b/Dockerfile +new file mode 100644 +index 0000000..993c83d +--- /dev/null ++++ b/Dockerfile +@@ -0,0 +1,11 @@ ++FROM node:18-alpine ++ ++WORKDIR /app ++ ++COPY package.json yarn.lock ./ ++RUN yarn install ++ ++COPY . . ++ ++EXPOSE 3000 ++CMD ["yarn", "start"] +diff --git a/scripts/publish.ts b/scripts/publish.ts +new file mode 100644 +index 0000000..966edcf +--- /dev/null ++++ b/scripts/publish.ts +@@ -0,0 +1,64 @@ ++import dotenv from 'dotenv' ++import { AtpAgent, BlobRef, AppBskyFeedDefs } from '@atproto/api' ++import fs from 'fs/promises' ++import { ids } from '../src/lexicon/lexicons' ++ ++const run = async () => { ++ dotenv.config() ++ ++ const handle = process.env.FEEDGEN_HANDLE ++ const password = process.env.FEEDGEN_PASSWORD ++ const recordName = process.env.FEEDGEN_RECORD_NAME || 'app' ++ const displayName = process.env.FEEDGEN_DISPLAY_NAME || 'App Feed' ++ const description = process.env.FEEDGEN_DESCRIPTION || 'Automated App Feed' ++ const avatar = process.env.FEEDGEN_AVATAR ++ const service = process.env.FEEDGEN_SERVICE_URL || 'https://syu.is' ++ ++ if (!handle || !password) { ++ throw new Error('Please provide FEEDGEN_HANDLE and FEEDGEN_PASSWORD environment variables') ++ } ++ ++ if (!process.env.FEEDGEN_SERVICE_DID && !process.env.FEEDGEN_HOSTNAME) { ++ throw new Error('Please provide a hostname in the .env file') ++ } ++ ++ const feedGenDid = ++ process.env.FEEDGEN_SERVICE_DID ?? `did:web:${process.env.FEEDGEN_HOSTNAME}` ++ ++ const agent = new AtpAgent({ service }) ++ await agent.login({ identifier: handle, password }) ++ ++ let avatarRef: BlobRef | undefined ++ if (avatar) { ++ let encoding: string ++ if (avatar.endsWith('png')) { ++ encoding = 'image/png' ++ } else if (avatar.endsWith('jpg') || avatar.endsWith('jpeg')) { ++ encoding = 'image/jpeg' ++ } else { ++ throw new Error('expected png or jpeg') ++ } ++ const img = await fs.readFile(avatar) ++ const blobRes = await agent.api.com.atproto.repo.uploadBlob(img, { ++ encoding, ++ }) ++ avatarRef = blobRes.data.blob ++ } ++ ++ await agent.api.com.atproto.repo.putRecord({ ++ repo: agent.session?.did ?? '', ++ collection: ids.AppBskyFeedGenerator, ++ rkey: recordName, ++ record: { ++ did: feedGenDid, ++ displayName: displayName, ++ description: description, ++ avatar: avatarRef, ++ createdAt: new Date().toISOString(), ++ }, ++ }) ++ ++ console.log('All done 🎉') ++} ++ ++run() +diff --git a/src/algos/app.ts b/src/algos/app.ts +new file mode 100644 +index 0000000..2376be9 +--- /dev/null ++++ b/src/algos/app.ts +@@ -0,0 +1,35 @@ ++import { QueryParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton' ++import { AppContext } from '../config' ++ ++// max 15 chars ++export const shortname = 'app' ++ ++export const handler = async (ctx: AppContext, params: QueryParams) => { ++ let builder = ctx.db ++ .selectFrom('post') ++ .selectAll() ++ .orderBy('indexedAt', 'desc') ++ .orderBy('cid', 'desc') ++ .limit(params.limit) ++ ++ if (params.cursor) { ++ const timeStr = new Date(parseInt(params.cursor, 10)).toISOString() ++ builder = builder.where('post.indexedAt', '<', timeStr) ++ } ++ const res = await builder.execute() ++ ++ const feed = res.map((row) => ({ ++ post: row.uri, ++ })) ++ ++ let cursor: string | undefined ++ const last = res.at(-1) ++ if (last) { ++ cursor = new Date(last.indexedAt).getTime().toString(10) ++ } ++ ++ return { ++ cursor, ++ feed, ++ } ++} +diff --git a/src/algos/index.ts b/src/algos/index.ts +index b7ee48a..102cb93 100644 +--- a/src/algos/index.ts ++++ b/src/algos/index.ts +@@ -4,11 +4,13 @@ import { + OutputSchema as AlgoOutput, + } from '../lexicon/types/app/bsky/feed/getFeedSkeleton' + import * as whatsAlf from './whats-alf' ++import * as app from './app' + + type AlgoHandler = (ctx: AppContext, params: QueryParams) => Promise + + const algos: Record = { + [whatsAlf.shortname]: whatsAlf.handler, ++ [app.shortname]: app.handler, + } + + export default algos +diff --git a/src/index.ts b/src/index.ts +index c3bd006..1e7f0b5 100644 +--- a/src/index.ts ++++ b/src/index.ts +@@ -24,6 +24,8 @@ const run = async () => { + console.log( + `🤖 running feed generator at http://${server.cfg.listenhost}:${server.cfg.port}`, + ) ++ console.log('Supported algos:', Object.keys(require('./algos').default)) ++ console.log('Publisher DID:', server.cfg.publisherDid) + } + + const maybeStr = (val?: string) => { +diff --git a/src/methods/feed-generation.ts b/src/methods/feed-generation.ts +index b887413..34c5148 100644 +--- a/src/methods/feed-generation.ts ++++ b/src/methods/feed-generation.ts +@@ -10,7 +10,7 @@ export default function (server: Server, ctx: AppContext) { + const feedUri = new AtUri(params.feed) + const algo = algos[feedUri.rkey] + if ( +- feedUri.hostname !== ctx.cfg.publisherDid || ++ //feedUri.hostname !== ctx.cfg.publisherDid || + feedUri.collection !== 'app.bsky.feed.generator' || + !algo + ) { +diff --git a/src/subscription.ts b/src/subscription.ts +index 0422a03..d591ef9 100644 +--- a/src/subscription.ts ++++ b/src/subscription.ts +@@ -19,10 +19,6 @@ export class FirehoseSubscription extends FirehoseSubscriptionBase { + + const postsToDelete = ops.posts.deletes.map((del) => del.uri) + const postsToCreate = ops.posts.creates +- .filter((create) => { +- // only alf-related posts +- return create.record.text.toLowerCase().includes('alf') +- }) + .map((create) => { + // map alf-related posts to a db row + return {