at/k8s/bsky.yaml

apiVersion: v1
kind: Service
metadata:
  name: bsky
  namespace: atproto
spec:
  selector:
    app: bsky
  ports:
    - name: api
      port: 2584
      targetPort: 2584
    - name: dataplane
      port: 3001
      targetPort: 3001
    - name: bsync
      port: 3002
      targetPort: 3002
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: bsky-data
  namespace: atproto
spec:
  accessModes: ["ReadWriteOnce"]
  resources:
    requests:
      storage: 5Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: bsky
  namespace: atproto
spec:
  replicas: 1
  selector:
    matchLabels:
      app: bsky
  template:
    metadata:
      labels:
        app: bsky
    spec:
      securityContext:
        runAsUser: 0
      containers:
        - name: bsky
          image: registry/bsky
          command: ["node", "--enable-source-maps", "api.js"]
          ports:
            - containerPort: 2584
            - containerPort: 3001
            - containerPort: 3002
          env:
            - name: BSKY_PORT
              value: "2584"
            - name: BSKY_BLOB_CACHE_LOC
              value: "/data/"
            - name: BSKY_BSYNC_HTTP_VERSION
              value: "1.1"
            - name: BSKY_BSYNC_PORT
              value: "3002"
            - name: BSKY_BSYNC_URL
              value: "http://localhost:3002"
            - name: BSKY_COURIER_URL
              value: "http://fake-courier.example.invalid/"
            - name: BSKY_DATAPLANE_HTTP_VERSION
              value: "1.1"
            - name: BSKY_DATAPLANE_PORT
              value: "3001"
            - name: BSKY_DATAPLANE_URLS
              value: "http://localhost:3001"
            - name: BSKY_DB_POSTGRES_URL
              value: "postgres://postgres:postgres@database/bsky"
            - name: BSKY_DID_PLC_URL
              value: "https://plc.syu.is"
            - name: BSKY_PUBLIC_URL
              value: "https://bsky.syu.is"
            - name: BSKY_REPO_PROVIDER
              value: "ws://bgs:2470"
            - name: BSKY_SERVER_DID
              value: "did:web:bsky.syu.is"
            - name: MOD_SERVICE_DID
              value: "did:web:ozone.syu.is"
            - name: BSKY_ADMIN_PASSWORDS
              valueFrom:
                secretKeyRef:
                  name: atproto-secrets
                  key: bsky-admin-passwords
            - name: BSKY_SERVICE_SIGNING_KEY
              valueFrom:
                secretKeyRef:
                  name: atproto-secrets
                  key: bsky-service-signing-key
          volumeMounts:
            - name: data
              mountPath: /data
          livenessProbe:
            httpGet:
              path: /xrpc/_health
              port: 2584
            initialDelaySeconds: 15
            periodSeconds: 10
            failureThreshold: 3
          readinessProbe:
            httpGet:
              path: /xrpc/_health
              port: 2584
            initialDelaySeconds: 5
            periodSeconds: 5
      volumes:
        - name: data
          persistentVolumeClaim:
            claimName: bsky-data
---
## bsky subscription monitor
## subscriptionが停止していたらPodを再起動する
apiVersion: batch/v1
kind: CronJob
metadata:
  name: bsky-subscription-watchdog
  namespace: atproto
spec:
  schedule: "*/5 * * * *"
  jobTemplate:
    spec:
      template:
        spec:
          containers:
            - name: watchdog
              image: postgres:16-alpine
              command:
                - /bin/sh
                - -c
                - |
                  # BGSの最新seqを取得
                  LATEST_SEQ=$(psql -t -A "$DB_URL" -c "SELECT COALESCE(MAX(seq),0) FROM repo_event_records")
                  # bskyのsubscription cursorを取得
                  BSKY_CURSOR=$(psql -t -A "$BSKY_DB_URL" -c "SELECT COALESCE(state,0) FROM subscription WHERE service='ws://bgs:2470' LIMIT 1")
                  LAG=$((LATEST_SEQ - BSKY_CURSOR))
                  echo "BGS seq=$LATEST_SEQ, bsky cursor=$BSKY_CURSOR, lag=$LAG"
                  if [ "$LAG" -gt 50 ]; then
                    echo "WARN: bsky subscription lag=$LAG, restarting bsky pod"
                    # Podを削除すればDeploymentが再作成する
                    apk add --no-cache curl > /dev/null 2>&1
                    APISERVER=https://kubernetes.default.svc
                    TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
                    NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)
                    # bsky podを取得して削除
                    POD=$(curl -s -k -H "Authorization: Bearer $TOKEN" \
                      "$APISERVER/api/v1/namespaces/$NAMESPACE/pods?labelSelector=app=bsky" \
                      | grep -o '"name":"bsky-[^"]*"' | head -1 | cut -d'"' -f4)
                    if [ -n "$POD" ]; then
                      curl -s -k -X DELETE -H "Authorization: Bearer $TOKEN" \
                        "$APISERVER/api/v1/namespaces/$NAMESPACE/pods/$POD"
                      echo "Deleted pod $POD"
                    fi
                  else
                    echo "OK: subscription is healthy"
                  fi
              env:
                - name: DB_URL
                  value: "postgres://postgres:postgres@database/bgs"
                - name: BSKY_DB_URL
                  value: "postgres://postgres:postgres@database/bsky"
          restartPolicy: OnFailure
          serviceAccountName: bsky-watchdog
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: bsky-watchdog
  namespace: atproto
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: bsky-watchdog
  namespace: atproto
rules:
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["get", "list", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: bsky-watchdog
  namespace: atproto
subjects:
  - kind: ServiceAccount
    name: bsky-watchdog
    namespace: atproto
roleRef:
  kind: Role
  name: bsky-watchdog
  apiGroup: rbac.authorization.k8s.io