Compare commits
2 Commits
main
...
8b30848de1
| Author | SHA1 | Date | |
|---|---|---|---|
8b30848de1
|
|||
|
cc4350be9b
|
1
.gitignore
vendored
1
.gitignore
vendored
@@ -2,3 +2,4 @@ repos
|
||||
.claude
|
||||
deploy.yml
|
||||
claude.md
|
||||
feed
|
||||
|
||||
35
compose.yml
35
compose.yml
@@ -98,18 +98,6 @@ services:
|
||||
- ./envs/social-app
|
||||
command: "/usr/bin/bskyweb serve"
|
||||
|
||||
jetstream:
|
||||
build:
|
||||
context: ./repos/jetstream/
|
||||
dockerfile: Dockerfile
|
||||
ports:
|
||||
- 6008:6008
|
||||
volumes:
|
||||
- ./data/jetstream:/data
|
||||
restart: always
|
||||
env_file:
|
||||
- ./envs/jetstream
|
||||
|
||||
ozone:
|
||||
ports:
|
||||
- 2585:3000
|
||||
@@ -143,3 +131,26 @@ services:
|
||||
depends_on:
|
||||
database:
|
||||
condition: service_healthy
|
||||
|
||||
jetstream:
|
||||
build:
|
||||
context: ./repos/jetstream/
|
||||
dockerfile: Dockerfile
|
||||
ports:
|
||||
- 6008:6008
|
||||
volumes:
|
||||
- ./data/jetstream:/data
|
||||
restart: always
|
||||
env_file:
|
||||
- ./envs/jetstream
|
||||
|
||||
feed:
|
||||
ports:
|
||||
- 2587:3000
|
||||
build:
|
||||
context: ./repos/feed-generator/
|
||||
restart: always
|
||||
env_file:
|
||||
- ./envs/feed
|
||||
volumes:
|
||||
- ./data/feed:/data
|
||||
|
||||
84
install.zsh
84
install.zsh
@@ -1,7 +1,9 @@
|
||||
#!/bin/zsh
|
||||
|
||||
function at-repos-env() {
|
||||
APP_PASSWORD=xxx
|
||||
host=syu.is
|
||||
handle=ai.syui.ai
|
||||
did=did:plc:6qyecktefllvenje24fcxnie
|
||||
repos=(
|
||||
"https://github.com/did-method-plc/did-method-plc"
|
||||
@@ -20,6 +22,12 @@ function at-repos-env() {
|
||||
"bgs"
|
||||
"ozone"
|
||||
"social-app"
|
||||
"feed"
|
||||
)
|
||||
handles=(
|
||||
"syui.syui.ai"
|
||||
"ai.syui.ai"
|
||||
"apple.syu.is"
|
||||
)
|
||||
d=${0:a:h}
|
||||
dh=${0:a:h:h}
|
||||
@@ -45,6 +53,8 @@ PATCH_FILES=(
|
||||
"disable-statsig-sdk.diff"
|
||||
"140-social-app-yarn-network-timeout.patch"
|
||||
"130-atproto-ozone-enable-daemon-v2.patch"
|
||||
"190-bgs-disable-ratelimit.patch"
|
||||
"200-feed-generator-custom.patch"
|
||||
)
|
||||
|
||||
function at-repos-clone() {
|
||||
@@ -114,6 +124,7 @@ function at-repos-social-app-avatar-write() {
|
||||
grep -R $did_admin .|cut -d : -f 1|sort -u|xargs sed -i "s/${did_admin}/${did}/g"
|
||||
}
|
||||
|
||||
|
||||
# Common patch function with status detection
|
||||
function apply-patch() {
|
||||
local patch_name=$1
|
||||
@@ -240,6 +251,10 @@ function at-repos-patch-apply-all() {
|
||||
repo="atproto"
|
||||
elif [[ $filename == *"pds"* ]]; then
|
||||
repo="atproto"
|
||||
elif [[ $filename == *"indigo"* || $filename == *"bgs"* ]]; then
|
||||
repo="indigo"
|
||||
elif [[ $filename == *"feed"* ]]; then
|
||||
repo="feed-generator"
|
||||
fi
|
||||
|
||||
patch-apply "$title" "$repo" "$filename"
|
||||
@@ -345,6 +360,67 @@ function at-repos-pull-docker() {
|
||||
docker compose up -d --pull always
|
||||
}
|
||||
|
||||
function at-repos-reset-bgs-db() {
|
||||
dp=at-database-1
|
||||
BGS_ADMIN_KEY=`cat $d/envs/bgs | grep BGS_ADMIN_KEY | cut -d '=' -f 2`
|
||||
|
||||
echo "🛑 Stopping BGS..."
|
||||
docker compose stop bgs
|
||||
|
||||
echo "🗑️ Cleaning data..."
|
||||
sudo rm -rf $d/data/bgs/*
|
||||
|
||||
echo "♻️ Resetting Databases..."
|
||||
docker exec -i $dp psql -U postgres -c "DROP DATABASE IF EXISTS bgs;"
|
||||
docker exec -i $dp psql -U postgres -c "CREATE DATABASE bgs;"
|
||||
docker exec -i $dp psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE bgs TO postgres;"
|
||||
|
||||
docker exec -i $dp psql -U postgres -c "DROP DATABASE IF EXISTS carstore;"
|
||||
docker exec -i $dp psql -U postgres -c "CREATE DATABASE carstore;"
|
||||
docker exec -i $dp psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE carstore TO postgres;"
|
||||
|
||||
echo "🚀 Starting BGS to initialize tables..."
|
||||
docker compose up -d bgs
|
||||
|
||||
echo "⏳ Waiting 10s for BGS migration..."
|
||||
sleep 10
|
||||
|
||||
echo "⚙️ Updating Slurp Config..."
|
||||
docker exec -i $dp psql -U postgres -d bgs -c "UPDATE slurp_configs SET new_subs_disabled = false, new_pds_per_day_limit = 1000 WHERE id = 1;"
|
||||
|
||||
echo "🔗 Registering Trusted Domain & Resetting Repos..."
|
||||
# Retry loop for addTrustedDomain as BGS might still be warming up
|
||||
for i in {1..5}; do
|
||||
if curl -f -X POST "https://bgs.${host}/admin/pds/addTrustedDomain?domain=${host}" -H "Authorization: Bearer ${BGS_ADMIN_KEY}"; then
|
||||
echo "✅ Trusted domain registered"
|
||||
break
|
||||
fi
|
||||
echo "Bot failed to contact BGS (attempt $i/5)... waiting 5s"
|
||||
sleep 5
|
||||
done
|
||||
|
||||
for ((i=1; i<=${#handles}; i++)); do
|
||||
handle=${handles[$i]}
|
||||
did=`curl -sL "https://${host}/xrpc/com.atproto.repo.describeRepo?repo=${handle}" |jq -r .did`
|
||||
if [ ! -z "$did" ] && [ "$did" != "null" ]; then
|
||||
echo "Resetting repo: $handle ($did)"
|
||||
curl -X POST "https://bgs.${host}/admin/repo/reset?did=${did}" \
|
||||
-H "Authorization: Bearer ${BGS_ADMIN_KEY}"
|
||||
else
|
||||
echo "Skipping reset for $handle (DID not found)"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function at-repos-feed-generator-start-push() {
|
||||
cd $d/repos/feed-generator
|
||||
yarn install
|
||||
FEEDGEN_HANDLE=${handle}
|
||||
FEEDGEN_PASSWORD=${APP_PASSWORD}
|
||||
FEEDGEN_RECORD_NAME=app
|
||||
npx tsx scripts/publish.ts
|
||||
}
|
||||
|
||||
at-repos-env
|
||||
case "$1" in
|
||||
pull)
|
||||
@@ -375,10 +451,18 @@ case "$1" in
|
||||
cd $d;docker compose down
|
||||
exit
|
||||
;;
|
||||
feed-push)
|
||||
at-repos-feed-generator-start-push
|
||||
exit
|
||||
;;
|
||||
esac
|
||||
|
||||
case "`cat /etc/hostname`" in
|
||||
at)
|
||||
if [ "$1" = "bgs-reset" ];then
|
||||
at-repos-reset-bgs-db
|
||||
exit
|
||||
fi
|
||||
at-repos-pull-docker
|
||||
exit
|
||||
;;
|
||||
|
||||
12
patching/190-bgs-disable-ratelimit.patch
Normal file
12
patching/190-bgs-disable-ratelimit.patch
Normal file
@@ -0,0 +1,12 @@
|
||||
diff --git a/bgs/fedmgr.go b/bgs/fedmgr.go
|
||||
index 2235c6e..e69de29 100644
|
||||
--- a/bgs/fedmgr.go
|
||||
+++ b/bgs/fedmgr.go
|
||||
@@ -342,6 +342,7 @@ var ErrNewSubsDisabled = fmt.Errorf("new subscriptions temporarily disabled")
|
||||
// Checks whether a host is allowed to be subscribed to
|
||||
// must be called with the slurper lock held
|
||||
func (s *Slurper) canSlurpHost(host string) bool {
|
||||
+ return true
|
||||
// Check if we're over the limit for new PDSs today
|
||||
if !s.NewPDSPerDayLimiter.Allow() {
|
||||
return false
|
||||
144
patching/200-feed-generator-custom.patch
Normal file
144
patching/200-feed-generator-custom.patch
Normal file
@@ -0,0 +1,144 @@
|
||||
diff --git a/scripts/publish.ts b/scripts/publish.ts
|
||||
new file mode 100644
|
||||
index 0000000..966edcf
|
||||
--- /dev/null
|
||||
+++ b/scripts/publish.ts
|
||||
@@ -0,0 +1,64 @@
|
||||
+import dotenv from 'dotenv'
|
||||
+import { AtpAgent, BlobRef, AppBskyFeedDefs } from '@atproto/api'
|
||||
+import fs from 'fs/promises'
|
||||
+import { ids } from '../src/lexicon/lexicons'
|
||||
+
|
||||
+const run = async () => {
|
||||
+ dotenv.config()
|
||||
+
|
||||
+ const handle = process.env.FEEDGEN_HANDLE
|
||||
+ const password = process.env.FEEDGEN_PASSWORD
|
||||
+ const recordName = process.env.FEEDGEN_RECORD_NAME || 'app'
|
||||
+ const displayName = process.env.FEEDGEN_DISPLAY_NAME || 'App Feed'
|
||||
+ const description = process.env.FEEDGEN_DESCRIPTION || 'Automated App Feed'
|
||||
+ const avatar = process.env.FEEDGEN_AVATAR
|
||||
+ const service = process.env.FEEDGEN_SERVICE_URL || 'https://syu.is'
|
||||
+
|
||||
+ if (!handle || !password) {
|
||||
+ throw new Error('Please provide FEEDGEN_HANDLE and FEEDGEN_PASSWORD environment variables')
|
||||
+ }
|
||||
+
|
||||
+ if (!process.env.FEEDGEN_SERVICE_DID && !process.env.FEEDGEN_HOSTNAME) {
|
||||
+ throw new Error('Please provide a hostname in the .env file')
|
||||
+ }
|
||||
+
|
||||
+ const feedGenDid =
|
||||
+ process.env.FEEDGEN_SERVICE_DID ?? `did:web:${process.env.FEEDGEN_HOSTNAME}`
|
||||
+
|
||||
+ const agent = new AtpAgent({ service })
|
||||
+ await agent.login({ identifier: handle, password })
|
||||
+
|
||||
+ let avatarRef: BlobRef | undefined
|
||||
+ if (avatar) {
|
||||
+ let encoding: string
|
||||
+ if (avatar.endsWith('png')) {
|
||||
+ encoding = 'image/png'
|
||||
+ } else if (avatar.endsWith('jpg') || avatar.endsWith('jpeg')) {
|
||||
+ encoding = 'image/jpeg'
|
||||
+ } else {
|
||||
+ throw new Error('expected png or jpeg')
|
||||
+ }
|
||||
+ const img = await fs.readFile(avatar)
|
||||
+ const blobRes = await agent.api.com.atproto.repo.uploadBlob(img, {
|
||||
+ encoding,
|
||||
+ })
|
||||
+ avatarRef = blobRes.data.blob
|
||||
+ }
|
||||
+
|
||||
+ await agent.api.com.atproto.repo.putRecord({
|
||||
+ repo: agent.session?.did ?? '',
|
||||
+ collection: ids.AppBskyFeedGenerator,
|
||||
+ rkey: recordName,
|
||||
+ record: {
|
||||
+ did: feedGenDid,
|
||||
+ displayName: displayName,
|
||||
+ description: description,
|
||||
+ avatar: avatarRef,
|
||||
+ createdAt: new Date().toISOString(),
|
||||
+ },
|
||||
+ })
|
||||
+
|
||||
+ console.log('All done 🎉')
|
||||
+}
|
||||
+
|
||||
+run()
|
||||
diff --git a/src/algos/app.ts b/src/algos/app.ts
|
||||
new file mode 100644
|
||||
index 0000000..2376be9
|
||||
--- /dev/null
|
||||
+++ b/src/algos/app.ts
|
||||
@@ -0,0 +1,35 @@
|
||||
+import { QueryParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton'
|
||||
+import { AppContext } from '../config'
|
||||
+
|
||||
+// max 15 chars
|
||||
+export const shortname = 'app'
|
||||
+
|
||||
+export const handler = async (ctx: AppContext, params: QueryParams) => {
|
||||
+ let builder = ctx.db
|
||||
+ .selectFrom('post')
|
||||
+ .selectAll()
|
||||
+ .orderBy('indexedAt', 'desc')
|
||||
+ .orderBy('cid', 'desc')
|
||||
+ .limit(params.limit)
|
||||
+
|
||||
+ if (params.cursor) {
|
||||
+ const timeStr = new Date(parseInt(params.cursor, 10)).toISOString()
|
||||
+ builder = builder.where('post.indexedAt', '<', timeStr)
|
||||
+ }
|
||||
+ const res = await builder.execute()
|
||||
+
|
||||
+ const feed = res.map((row) => ({
|
||||
+ post: row.uri,
|
||||
+ }))
|
||||
+
|
||||
+ let cursor: string | undefined
|
||||
+ const last = res.at(-1)
|
||||
+ if (last) {
|
||||
+ cursor = new Date(last.indexedAt).getTime().toString(10)
|
||||
+ }
|
||||
+
|
||||
+ return {
|
||||
+ cursor,
|
||||
+ feed,
|
||||
+ }
|
||||
+}
|
||||
diff --git a/src/algos/index.ts b/src/algos/index.ts
|
||||
index b7ee48a..102cb93 100644
|
||||
--- a/src/algos/index.ts
|
||||
+++ b/src/algos/index.ts
|
||||
@@ -4,11 +4,13 @@ import {
|
||||
OutputSchema as AlgoOutput,
|
||||
} from '../lexicon/types/app/bsky/feed/getFeedSkeleton'
|
||||
import * as whatsAlf from './whats-alf'
|
||||
+import * as app from './app'
|
||||
|
||||
type AlgoHandler = (ctx: AppContext, params: QueryParams) => Promise<AlgoOutput>
|
||||
|
||||
const algos: Record<string, AlgoHandler> = {
|
||||
[whatsAlf.shortname]: whatsAlf.handler,
|
||||
+ [app.shortname]: app.handler,
|
||||
}
|
||||
|
||||
export default algos
|
||||
diff --git a/src/subscription.ts b/src/subscription.ts
|
||||
index 0422a03..d591ef9 100644
|
||||
--- a/src/subscription.ts
|
||||
+++ b/src/subscription.ts
|
||||
@@ -19,10 +19,6 @@ export class FirehoseSubscription extends FirehoseSubscriptionBase {
|
||||
|
||||
const postsToDelete = ops.posts.deletes.map((del) => del.uri)
|
||||
const postsToCreate = ops.posts.creates
|
||||
- .filter((create) => {
|
||||
- // only alf-related posts
|
||||
- return create.record.text.toLowerCase().includes('alf')
|
||||
- })
|
||||
.map((create) => {
|
||||
// map alf-related posts to a db row
|
||||
return {
|
||||
Reference in New Issue
Block a user