add feed

.gitignore

@@ -2,3 +2,4 @@ repos
 .claude
 deploy.yml
 claude.md
+feed

compose.yml

@@ -98,18 +98,6 @@ services:
       - ./envs/social-app
     command: "/usr/bin/bskyweb serve"
 
-  jetstream:
-    build:
-      context: ./repos/jetstream/
-      dockerfile: Dockerfile
-    ports:
-      - 6008:6008
-    volumes:
-      - ./data/jetstream:/data
-    restart: always
-    env_file:
-      - ./envs/jetstream
-
   ozone:
     ports:
       - 2585:3000
@@ -143,3 +131,26 @@ services:
     depends_on:
       database:
         condition: service_healthy
+
+  jetstream:
+    build:
+      context: ./repos/jetstream/
+      dockerfile: Dockerfile
+    ports:
+      - 6008:6008
+    volumes:
+      - ./data/jetstream:/data
+    restart: always
+    env_file:
+      - ./envs/jetstream
+
+  feed:
+    ports:
+      - 2587:3000
+    build:
+      context: ./repos/feed-generator/
+    restart: always
+    env_file:
+      - ./envs/feed
+    volumes:
+      - ./data/feed:/data

install.zsh

@@ -1,7 +1,9 @@
 #!/bin/zsh
 
 function at-repos-env() {
+	APP_PASSWORD=xxx
 	host=syu.is
+	handle=ai.syui.ai
 	did=did:plc:6qyecktefllvenje24fcxnie
 	repos=(
 		"https://github.com/did-method-plc/did-method-plc"
@@ -20,6 +22,12 @@ function at-repos-env() {
 		"bgs"
 		"ozone"
 		"social-app"
+		"feed"
+	)
+	handles=(
+		"syui.syui.ai"
+		"ai.syui.ai"
+		"apple.syu.is"
 	)
 	d=${0:a:h}
 	dh=${0:a:h:h}
@@ -45,6 +53,8 @@ PATCH_FILES=(
 	"disable-statsig-sdk.diff"
 	"140-social-app-yarn-network-timeout.patch"
 	"130-atproto-ozone-enable-daemon-v2.patch"
+	"190-bgs-disable-ratelimit.patch"
+	"200-feed-generator-custom.patch"
 )
 
 function at-repos-clone() {
@@ -114,6 +124,7 @@ function at-repos-social-app-avatar-write() {
 	grep -R $did_admin .|cut -d : -f 1|sort -u|xargs sed -i "s/${did_admin}/${did}/g"
 }
 
+
 # Common patch function with status detection
 function apply-patch() {
 	local patch_name=$1
@@ -240,6 +251,10 @@ function at-repos-patch-apply-all() {
 			repo="atproto"
 		elif [[ $filename == *"pds"* ]]; then
 			repo="atproto"
+		elif [[ $filename == *"indigo"* || $filename == *"bgs"* ]]; then
+			repo="indigo"
+		elif [[ $filename == *"feed"* ]]; then
+			repo="feed-generator"
 		fi
 
 		patch-apply "$title" "$repo" "$filename"
@@ -345,6 +360,67 @@ function at-repos-pull-docker() {
 	docker compose up -d --pull always
 }
 
+function at-repos-reset-bgs-db() {
+	dp=at-database-1
+	BGS_ADMIN_KEY=`cat $d/envs/bgs | grep BGS_ADMIN_KEY | cut -d '=' -f 2`
+	
+	echo "🛑 Stopping BGS..."
+	docker compose stop bgs
+	
+	echo "🗑️  Cleaning data..."
+	sudo rm -rf $d/data/bgs/*
+	
+	echo "♻️  Resetting Databases..."
+	docker exec -i $dp psql -U postgres -c "DROP DATABASE IF EXISTS bgs;"
+	docker exec -i $dp psql -U postgres -c "CREATE DATABASE bgs;"
+	docker exec -i $dp psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE bgs TO postgres;"
+	
+	docker exec -i $dp psql -U postgres -c "DROP DATABASE IF EXISTS carstore;"
+	docker exec -i $dp psql -U postgres -c "CREATE DATABASE carstore;"
+	docker exec -i $dp psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE carstore TO postgres;"
+
+	echo "🚀 Starting BGS to initialize tables..."
+	docker compose up -d bgs
+	
+	echo "⏳ Waiting 10s for BGS migration..."
+	sleep 10
+	
+	echo "⚙️  Updating Slurp Config..."
+	docker exec -i $dp psql -U postgres -d bgs -c "UPDATE slurp_configs SET new_subs_disabled = false, new_pds_per_day_limit = 1000 WHERE id = 1;"
+
+	echo "🔗 Registering Trusted Domain & Resetting Repos..."
+	# Retry loop for addTrustedDomain as BGS might still be warming up
+	for i in {1..5}; do
+		if curl -f -X POST "https://bgs.${host}/admin/pds/addTrustedDomain?domain=${host}" -H "Authorization: Bearer ${BGS_ADMIN_KEY}"; then
+			echo "✅ Trusted domain registered"
+			break
+		fi
+		echo "Bot failed to contact BGS (attempt $i/5)... waiting 5s"
+		sleep 5
+	done
+
+	for ((i=1; i<=${#handles}; i++)); do
+		handle=${handles[$i]}
+		did=`curl -sL "https://${host}/xrpc/com.atproto.repo.describeRepo?repo=${handle}" |jq -r .did`
+		if [ ! -z "$did" ] && [ "$did" != "null" ]; then
+			echo "Resetting repo: $handle ($did)"
+			curl -X POST "https://bgs.${host}/admin/repo/reset?did=${did}" \
+				-H "Authorization: Bearer ${BGS_ADMIN_KEY}" 
+		else
+			echo "Skipping reset for $handle (DID not found)"
+		fi
+	done
+}
+
+function at-repos-feed-generator-start-push() {
+	cd $d/repos/feed-generator
+	yarn install
+	FEEDGEN_HANDLE=${handle}
+	FEEDGEN_PASSWORD=${APP_PASSWORD}
+	FEEDGEN_RECORD_NAME=app
+	npx tsx scripts/publish.ts
+}
+
 at-repos-env
 case "$1" in 
 	pull)
@@ -375,10 +451,18 @@ case "$1" in
 		cd $d;docker compose down
 		exit
 		;;
+	feed-push)
+		at-repos-feed-generator-start-push
+		exit
+		;;
 esac
 
 case "`cat /etc/hostname`" in
 	at)
+		if [ "$1" = "bgs-reset" ];then
+			at-repos-reset-bgs-db
+			exit
+		fi
 		at-repos-pull-docker
 		exit
 		;;

patching/190-bgs-disable-ratelimit.patch

@@ -0,0 +1,12 @@
+diff --git a/bgs/fedmgr.go b/bgs/fedmgr.go
+index 2235c6e..e69de29 100644
+--- a/bgs/fedmgr.go
++++ b/bgs/fedmgr.go
+@@ -342,6 +342,7 @@ var ErrNewSubsDisabled = fmt.Errorf("new subscriptions temporarily disabled")
+ // Checks whether a host is allowed to be subscribed to
+ // must be called with the slurper lock held
+ func (s *Slurper) canSlurpHost(host string) bool {
++	return true
+ 	// Check if we're over the limit for new PDSs today
+ 	if !s.NewPDSPerDayLimiter.Allow() {
+ 		return false

patching/200-feed-generator-custom.patch

@@ -0,0 +1,144 @@
+diff --git a/scripts/publish.ts b/scripts/publish.ts
+new file mode 100644
+index 0000000..966edcf
+--- /dev/null
++++ b/scripts/publish.ts
+@@ -0,0 +1,64 @@
++import dotenv from 'dotenv'
++import { AtpAgent, BlobRef, AppBskyFeedDefs } from '@atproto/api'
++import fs from 'fs/promises'
++import { ids } from '../src/lexicon/lexicons'
++
++const run = async () => {
++    dotenv.config()
++
++    const handle = process.env.FEEDGEN_HANDLE
++    const password = process.env.FEEDGEN_PASSWORD
++    const recordName = process.env.FEEDGEN_RECORD_NAME || 'app'
++    const displayName = process.env.FEEDGEN_DISPLAY_NAME || 'App Feed'
++    const description = process.env.FEEDGEN_DESCRIPTION || 'Automated App Feed'
++    const avatar = process.env.FEEDGEN_AVATAR
++    const service = process.env.FEEDGEN_SERVICE_URL || 'https://syu.is'
++
++    if (!handle || !password) {
++        throw new Error('Please provide FEEDGEN_HANDLE and FEEDGEN_PASSWORD environment variables')
++    }
++
++    if (!process.env.FEEDGEN_SERVICE_DID && !process.env.FEEDGEN_HOSTNAME) {
++        throw new Error('Please provide a hostname in the .env file')
++    }
++
++    const feedGenDid =
++        process.env.FEEDGEN_SERVICE_DID ?? `did:web:${process.env.FEEDGEN_HOSTNAME}`
++
++    const agent = new AtpAgent({ service })
++    await agent.login({ identifier: handle, password })
++
++    let avatarRef: BlobRef | undefined
++    if (avatar) {
++        let encoding: string
++        if (avatar.endsWith('png')) {
++            encoding = 'image/png'
++        } else if (avatar.endsWith('jpg') || avatar.endsWith('jpeg')) {
++            encoding = 'image/jpeg'
++        } else {
++            throw new Error('expected png or jpeg')
++        }
++        const img = await fs.readFile(avatar)
++        const blobRes = await agent.api.com.atproto.repo.uploadBlob(img, {
++            encoding,
++        })
++        avatarRef = blobRes.data.blob
++    }
++
++    await agent.api.com.atproto.repo.putRecord({
++        repo: agent.session?.did ?? '',
++        collection: ids.AppBskyFeedGenerator,
++        rkey: recordName,
++        record: {
++            did: feedGenDid,
++            displayName: displayName,
++            description: description,
++            avatar: avatarRef,
++            createdAt: new Date().toISOString(),
++        },
++    })
++
++    console.log('All done 🎉')
++}
++
++run()
+diff --git a/src/algos/app.ts b/src/algos/app.ts
+new file mode 100644
+index 0000000..2376be9
+--- /dev/null
++++ b/src/algos/app.ts
+@@ -0,0 +1,35 @@
++import { QueryParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton'
++import { AppContext } from '../config'
++
++// max 15 chars
++export const shortname = 'app'
++
++export const handler = async (ctx: AppContext, params: QueryParams) => {
++    let builder = ctx.db
++        .selectFrom('post')
++        .selectAll()
++        .orderBy('indexedAt', 'desc')
++        .orderBy('cid', 'desc')
++        .limit(params.limit)
++
++    if (params.cursor) {
++        const timeStr = new Date(parseInt(params.cursor, 10)).toISOString()
++        builder = builder.where('post.indexedAt', '<', timeStr)
++    }
++    const res = await builder.execute()
++
++    const feed = res.map((row) => ({
++        post: row.uri,
++    }))
++
++    let cursor: string | undefined
++    const last = res.at(-1)
++    if (last) {
++        cursor = new Date(last.indexedAt).getTime().toString(10)
++    }
++
++    return {
++        cursor,
++        feed,
++    }
++}
+diff --git a/src/algos/index.ts b/src/algos/index.ts
+index b7ee48a..102cb93 100644
+--- a/src/algos/index.ts
++++ b/src/algos/index.ts
+@@ -4,11 +4,13 @@ import {
+   OutputSchema as AlgoOutput,
+ } from '../lexicon/types/app/bsky/feed/getFeedSkeleton'
+ import * as whatsAlf from './whats-alf'
++import * as app from './app'
+ 
+ type AlgoHandler = (ctx: AppContext, params: QueryParams) => Promise<AlgoOutput>
+ 
+ const algos: Record<string, AlgoHandler> = {
+   [whatsAlf.shortname]: whatsAlf.handler,
++  [app.shortname]: app.handler,
+ }
+ 
+ export default algos
+diff --git a/src/subscription.ts b/src/subscription.ts
+index 0422a03..d591ef9 100644
+--- a/src/subscription.ts
++++ b/src/subscription.ts
+@@ -19,10 +19,6 @@ export class FirehoseSubscription extends FirehoseSubscriptionBase {
+ 
+     const postsToDelete = ops.posts.deletes.map((del) => del.uri)
+     const postsToCreate = ops.posts.creates
+-      .filter((create) => {
+-        // only alf-related posts
+-        return create.record.text.toLowerCase().includes('alf')
+-      })
+       .map((create) => {
+         // map alf-related posts to a db row
+         return {