ai/log
1
0
Fork 0
Code Pull Requests Actions Wiki Activity

test update

Browse Source
This commit is contained in:
syui
2025-06-15 15:06:50 +09:00
parent 67b241f1e8
commit c12d42882c
16 changed files with 1363 additions and 742 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
src/commands/auth.rs
View File

@ -28,8 +28,31 @@ pub struct JetstreamConfig {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CollectionConfig {
pub comment: String,
pub user: String,
pub base: String, // Base collection name like "ai.syui.log"
}
impl CollectionConfig {
// Collection name builders
pub fn comment(&self) -> String {
self.base.clone()
}
pub fn user(&self) -> String {
format!("{}.user", self.base)
}
#[allow(dead_code)]
pub fn chat(&self) -> String {
format!("{}.chat", self.base)
}
pub fn chat_lang(&self) -> String {
format!("{}.chat.lang", self.base)
}
pub fn chat_comment(&self) -> String {
format!("{}.chat.comment", self.base)
}
}
impl Default for AuthConfig {
@ -47,8 +70,7 @@ impl Default for AuthConfig {
collections: vec!["ai.syui.log".to_string()],
},
collections: CollectionConfig {
comment: "ai.syui.log".to_string(),
user: "ai.syui.log.user".to_string(),
base: "ai.syui.log".to_string(),
},
}
}
@ -220,11 +242,50 @@ pub fn load_config() -> Result<AuthConfig> {
}
let config_json = fs::read_to_string(&config_path)?;
let mut config: AuthConfig = serde_json::from_str(&config_json)?;
// Update collection configuration
// Try to load as new format first, then migrate if needed
match serde_json::from_str::<AuthConfig>(&config_json) {
Ok(mut config) => {
// Update collection configuration
update_config_collections(&mut config);
Ok(config)
}
Err(e) => {
println!("{}", format!("Parse error: {}, attempting migration...", e).yellow());
// Try to migrate from old format
migrate_config_if_needed(&config_path, &config_json)
}
}
}
fn migrate_config_if_needed(config_path: &std::path::Path, config_json: &str) -> Result<AuthConfig> {
// Try to parse as old format and migrate to new simple format
let mut old_config: serde_json::Value = serde_json::from_str(config_json)?;
// Migrate old collections structure to new base-only structure
if let Some(collections) = old_config.get_mut("collections") {
// Extract base collection name from comment field or use default
let base_collection = collections.get("comment")
.and_then(|v| v.as_str())
.unwrap_or("ai.syui.log")
.to_string();
// Replace entire collections structure with new format
old_config["collections"] = serde_json::json!({
"base": base_collection
});
}
// Save migrated config
let migrated_config_json = serde_json::to_string_pretty(&old_config)?;
fs::write(config_path, migrated_config_json)?;
// Parse as new format
let mut config: AuthConfig = serde_json::from_value(old_config)?;
update_config_collections(&mut config);
println!("{}", "✅ Configuration migrated to new simplified format".green());
Ok(config)
}
@ -259,7 +320,7 @@ async fn test_api_access_with_auth(config: &AuthConfig) -> Result<()> {
let url = format!("{}/xrpc/com.atproto.repo.listRecords?repo={}&collection={}&limit=1",
config.admin.pds,
urlencoding::encode(&config.admin.did),
urlencoding::encode(&config.collections.comment));
urlencoding::encode(&config.collections.comment()));
let response = client
.get(&url)
@ -311,23 +372,14 @@ fn save_config(config: &AuthConfig) -> Result<()> {
Ok(())
}
// Generate collection names from admin DID or environment
// Generate collection config from environment
fn generate_collection_config() -> CollectionConfig {
// Check environment variables first
if let (Ok(comment), Ok(user)) = (
std::env::var("AILOG_COLLECTION_COMMENT"),
std::env::var("AILOG_COLLECTION_USER")
) {
return CollectionConfig {
comment,
user,
};
}
// Use VITE_OAUTH_COLLECTION for unified configuration
let base = std::env::var("VITE_OAUTH_COLLECTION")
.unwrap_or_else(|_| "ai.syui.log".to_string());
// Default collections
CollectionConfig {
comment: "ai.syui.log".to_string(),
user: "ai.syui.log.user".to_string(),
base,
}
}
@ -335,5 +387,5 @@ fn generate_collection_config() -> CollectionConfig {
pub fn update_config_collections(config: &mut AuthConfig) {
config.collections = generate_collection_config();
// Also update jetstream collections to monitor the comment collection
config.jetstream.collections = vec![config.collections.comment.clone()];
config.jetstream.collections = vec![config.collections.comment()];
}

28
src/commands/oauth.rs
View File

@ -45,18 +45,10 @@ pub async fn build(project_dir: PathBuf) -> Result<()> {
.and_then(|v| v.as_str())
.ok_or_else(|| anyhow::anyhow!("No admin DID found in [oauth] section"))?;
let collection_comment = oauth_config.get("collection_comment")
let collection_base = oauth_config.get("collection")
.and_then(|v| v.as_str())
.unwrap_or("ai.syui.log");
let collection_user = oauth_config.get("collection_user")
.and_then(|v| v.as_str())
.unwrap_or("ai.syui.log.user");
let collection_chat = oauth_config.get("collection_chat")
.and_then(|v| v.as_str())
.unwrap_or("ai.syui.log.chat");
// Extract AI config if present
let ai_config = config.get("ai")
.and_then(|v| v.as_table());
@ -109,15 +101,8 @@ VITE_OAUTH_CLIENT_ID={}/{}
VITE_OAUTH_REDIRECT_URI={}/{}
VITE_ADMIN_DID={}
# Collection names for OAuth app
VITE_COLLECTION_COMMENT={}
VITE_COLLECTION_USER={}
VITE_COLLECTION_CHAT={}
# Collection names for ailog (backward compatibility)
AILOG_COLLECTION_COMMENT={}
AILOG_COLLECTION_USER={}
AILOG_COLLECTION_CHAT={}
# Base collection for OAuth app and ailog (all others are derived)
VITE_OAUTH_COLLECTION={}
# AI Configuration
VITE_AI_ENABLED={}
@ -135,12 +120,7 @@ VITE_BSKY_PUBLIC_API={}
base_url, client_id_path,
base_url, redirect_path,
admin_did,
collection_comment,
collection_user,
collection_chat,
collection_comment,
collection_user,
collection_chat,
collection_base,
ai_enabled,
ai_ask_ai,
ai_provider,

366
src/commands/stream.rs
View File

@ -10,18 +10,58 @@ use std::process::{Command, Stdio};
use tokio::time::{sleep, Duration, interval};
use tokio_tungstenite::{connect_async, tungstenite::Message};
use toml;
use reqwest;
use super::auth::{load_config, load_config_with_refresh, AuthConfig};
#[derive(Debug, Deserialize)]
#[allow(dead_code)]
struct BlogPost {
title: String,
href: String,
#[serde(rename = "formated_time")]
#[allow(dead_code)]
date: String,
#[allow(dead_code)]
tags: Vec<String>,
#[allow(dead_code)]
contents: String,
}
#[derive(Debug, Deserialize)]
#[allow(dead_code)]
struct BlogIndex {
#[allow(dead_code)]
posts: Vec<BlogPost>,
}
#[derive(Debug, Serialize)]
struct OllamaRequest {
model: String,
prompt: String,
stream: bool,
options: OllamaOptions,
}
#[derive(Debug, Serialize)]
struct OllamaOptions {
temperature: f32,
top_p: f32,
num_predict: i32,
}
#[derive(Debug, Deserialize)]
struct OllamaResponse {
response: String,
}
// Load collection config with priority: env vars > project config.toml > defaults
fn load_collection_config(project_dir: Option<&Path>) -> Result<(String, String)> {
// 1. Check environment variables first (highest priority)
if let (Ok(comment), Ok(user)) = (
std::env::var("AILOG_COLLECTION_COMMENT"),
std::env::var("AILOG_COLLECTION_USER")
) {
if let Ok(base_collection) = std::env::var("VITE_OAUTH_COLLECTION") {
println!("{}", "📂 Using collection config from environment variables".cyan());
return Ok((comment, user));
let collection_user = format!("{}.user", base_collection);
return Ok((base_collection, collection_user));
}
// 2. Try to load from project config.toml (second priority)
@ -60,17 +100,16 @@ fn load_collection_config_from_project(project_dir: &Path) -> Result<(String, St
.and_then(|v| v.as_table())
.ok_or_else(|| anyhow::anyhow!("No [oauth] section found in config.toml"))?;
let collection_comment = oauth_config.get("collection_comment")
// Use new simplified collection structure (base collection)
let collection_base = oauth_config.get("collection")
.and_then(|v| v.as_str())
.unwrap_or("ai.syui.log")
.to_string();
let collection_user = oauth_config.get("collection_user")
.and_then(|v| v.as_str())
.unwrap_or("ai.syui.log.user")
.to_string();
// Derive user collection from base
let collection_user = format!("{}.user", collection_base);
Ok((collection_comment, collection_user))
Ok((collection_base, collection_user))
}
#[derive(Debug, Serialize, Deserialize)]
@ -118,15 +157,14 @@ fn get_pid_file() -> Result<PathBuf> {
Ok(pid_dir.join("stream.pid"))
}
pub async fn start(project_dir: Option<PathBuf>, daemon: bool) -> Result<()> {
pub async fn start(project_dir: Option<PathBuf>, daemon: bool, ai_generate: bool) -> Result<()> {
let mut config = load_config_with_refresh().await?;
// Load collection config with priority: env vars > project config > defaults
let (collection_comment, collection_user) = load_collection_config(project_dir.as_deref())?;
let (collection_comment, _collection_user) = load_collection_config(project_dir.as_deref())?;
// Update config with loaded collections
config.collections.comment = collection_comment.clone();
config.collections.user = collection_user;
config.collections.base = collection_comment.clone();
config.jetstream.collections = vec![collection_comment];
let pid_file = get_pid_file()?;
@ -151,6 +189,11 @@ pub async fn start(project_dir: Option<PathBuf>, daemon: bool) -> Result<()> {
args.push(project_path.to_string_lossy().to_string());
}
// Add ai_generate flag if enabled
if ai_generate {
args.push("--ai-generate".to_string());
}
let child = Command::new(current_exe)
.args(&args)
.stdin(Stdio::null())
@ -192,6 +235,19 @@ pub async fn start(project_dir: Option<PathBuf>, daemon: bool) -> Result<()> {
let max_reconnect_attempts = 10;
let mut config = config; // Make config mutable for token refresh
// Start AI generation monitor if enabled
if ai_generate {
let ai_config = config.clone();
tokio::spawn(async move {
loop {
if let Err(e) = run_ai_generation_monitor(&ai_config).await {
println!("{}", format!("❌ AI generation monitor error: {}", e).red());
sleep(Duration::from_secs(60)).await; // Wait 1 minute before retry
}
}
});
}
loop {
match run_monitor(&mut config).await {
Ok(_) => {
@ -344,7 +400,7 @@ async fn handle_message(text: &str, config: &mut AuthConfig) -> Result<()> {
if let (Some(collection), Some(commit), Some(did)) =
(&message.collection, &message.commit, &message.did) {
if collection == &config.collections.comment && commit.operation.as_deref() == Some("create") {
if collection == &config.collections.comment() && commit.operation.as_deref() == Some("create") {
let unknown_uri = "unknown".to_string();
let uri = commit.uri.as_ref().unwrap_or(&unknown_uri);
@ -438,7 +494,7 @@ async fn get_current_user_list(config: &mut AuthConfig) -> Result<Vec<UserRecord
let url = format!("{}/xrpc/com.atproto.repo.listRecords?repo={}&collection={}&limit=10",
config.admin.pds,
urlencoding::encode(&config.admin.did),
urlencoding::encode(&config.collections.user));
urlencoding::encode(&config.collections.user()));
let response = client
.get(&url)
@ -501,7 +557,7 @@ async fn post_user_list(config: &mut AuthConfig, users: &[UserRecord], metadata:
let rkey = format!("{}-{}", short_did, now.format("%Y-%m-%dT%H-%M-%S-%3fZ").to_string().replace(".", "-"));
let record = UserListRecord {
record_type: config.collections.user.clone(),
record_type: config.collections.user(),
users: users.to_vec(),
created_at: now.to_rfc3339(),
updated_by: UserInfo {
@ -515,7 +571,7 @@ async fn post_user_list(config: &mut AuthConfig, users: &[UserRecord], metadata:
let request_body = json!({
"repo": config.admin.did,
"collection": config.collections.user,
"collection": config.collections.user(),
"rkey": rkey,
"record": record
});
@ -759,7 +815,7 @@ async fn get_recent_comments(config: &mut AuthConfig) -> Result<Vec<Value>> {
let url = format!("{}/xrpc/com.atproto.repo.listRecords?repo={}&collection={}&limit=20",
config.admin.pds,
urlencoding::encode(&config.admin.did),
urlencoding::encode(&config.collections.comment));
urlencoding::encode(&config.collections.comment()));
if std::env::var("AILOG_DEBUG").is_ok() {
println!("{}", format!("🌐 API Request URL: {}", url).yellow());
@ -840,7 +896,7 @@ pub async fn test_api() -> Result<()> {
println!("{}", format!("✅ Successfully retrieved {} comments", comments.len()).green());
if comments.is_empty() {
println!("{}", format!("ℹ️ No comments found in {} collection", config.collections.comment).blue());
println!("{}", format!("ℹ️ No comments found in {} collection", config.collections.comment()).blue());
println!("💡 Try posting a comment first using the web interface");
} else {
println!("{}", "📝 Comment details:".cyan());
@ -871,5 +927,273 @@ pub async fn test_api() -> Result<()> {
}
}
Ok(())
}
// AI content generation functions
async fn generate_ai_content(content: &str, prompt_type: &str, ollama_host: &str) -> Result<String> {
let model = "gemma3:4b";
let prompt = match prompt_type {
"translate" => format!("Translate the following Japanese blog post to English. Keep the technical terms and code blocks intact:\n\n{}", content),
"comment" => format!("Read this blog post and provide an insightful comment about it. Focus on the key points and add your perspective:\n\n{}", content),
_ => return Err(anyhow::anyhow!("Unknown prompt type: {}", prompt_type)),
};
let request = OllamaRequest {
model: model.to_string(),
prompt,
stream: false,
options: OllamaOptions {
temperature: 0.9,
top_p: 0.9,
num_predict: 500,
},
};
let client = reqwest::Client::new();
// Try localhost first (for same-server deployment)
let localhost_url = "http://localhost:11434/api/generate";
match client.post(localhost_url).json(&request).send().await {
Ok(response) if response.status().is_success() => {
let ollama_response: OllamaResponse = response.json().await?;
println!("{}", "✅ Used localhost Ollama".green());
return Ok(ollama_response.response);
}
_ => {
println!("{}", "⚠️ Localhost Ollama not available, trying remote...".yellow());
}
}
// Fallback to remote host
let remote_url = format!("{}/api/generate", ollama_host);
let response = client.post(&remote_url).json(&request).send().await?;
if !response.status().is_success() {
return Err(anyhow::anyhow!("Ollama API request failed: {}", response.status()));
}
let ollama_response: OllamaResponse = response.json().await?;
println!("{}", "✅ Used remote Ollama".green());
Ok(ollama_response.response)
}
async fn run_ai_generation_monitor(config: &AuthConfig) -> Result<()> {
let blog_host = "https://syui.ai"; // TODO: Load from config
let ollama_host = "https://ollama.syui.ai"; // TODO: Load from config
let ai_did = "did:plc:4hqjfn7m6n5hno3doamuhgef"; // TODO: Load from config
println!("{}", "🤖 Starting AI content generation monitor...".cyan());
println!("📡 Blog host: {}", blog_host);
println!("🧠 Ollama host: {}", ollama_host);
println!("🤖 AI DID: {}", ai_did);
println!();
let mut interval = interval(Duration::from_secs(300)); // Check every 5 minutes
let client = reqwest::Client::new();
loop {
interval.tick().await;
println!("{}", "🔍 Checking for new blog posts...".blue());
match check_and_process_new_posts(&client, config, blog_host, ollama_host, ai_did).await {
Ok(count) => {
if count > 0 {
println!("{}", format!("✅ Processed {} new posts", count).green());
} else {
println!("{}", "ℹ️ No new posts found".blue());
}
}
Err(e) => {
println!("{}", format!("❌ Error processing posts: {}", e).red());
}
}
println!("{}", "⏰ Waiting for next check...".cyan());
}
}
async fn check_and_process_new_posts(
client: &reqwest::Client,
config: &AuthConfig,
blog_host: &str,
ollama_host: &str,
ai_did: &str,
) -> Result<usize> {
// Fetch blog index
let index_url = format!("{}/index.json", blog_host);
let response = client.get(&index_url).send().await?;
if !response.status().is_success() {
return Err(anyhow::anyhow!("Failed to fetch blog index: {}", response.status()));
}
let blog_posts: Vec<BlogPost> = response.json().await?;
println!("{}", format!("📄 Found {} posts in blog index", blog_posts.len()).cyan());
// Get existing AI generated content from collections
let existing_lang_records = get_existing_records(config, &config.collections.chat_lang()).await?;
let existing_comment_records = get_existing_records(config, &config.collections.chat_comment()).await?;
let mut processed_count = 0;
for post in blog_posts {
let post_slug = extract_slug_from_url(&post.href);
// Check if translation already exists
let translation_exists = existing_lang_records.iter().any(|record| {
record.get("value")
.and_then(|v| v.get("post_slug"))
.and_then(|s| s.as_str())
== Some(&post_slug)
});
// Check if comment already exists
let comment_exists = existing_comment_records.iter().any(|record| {
record.get("value")
.and_then(|v| v.get("post_slug"))
.and_then(|s| s.as_str())
== Some(&post_slug)
});
// Generate translation if not exists
if !translation_exists {
match generate_and_store_translation(client, config, &post, ollama_host, ai_did).await {
Ok(_) => {
println!("{}", format!("✅ Generated translation for: {}", post.title).green());
processed_count += 1;
}
Err(e) => {
println!("{}", format!("❌ Failed to generate translation for {}: {}", post.title, e).red());
}
}
}
// Generate comment if not exists
if !comment_exists {
match generate_and_store_comment(client, config, &post, ollama_host, ai_did).await {
Ok(_) => {
println!("{}", format!("✅ Generated comment for: {}", post.title).green());
processed_count += 1;
}
Err(e) => {
println!("{}", format!("❌ Failed to generate comment for {}: {}", post.title, e).red());
}
}
}
}
Ok(processed_count)
}
async fn get_existing_records(config: &AuthConfig, collection: &str) -> Result<Vec<serde_json::Value>> {
let client = reqwest::Client::new();
let url = format!("{}/xrpc/com.atproto.repo.listRecords?repo={}&collection={}&limit=100",
config.admin.pds,
urlencoding::encode(&config.admin.did),
urlencoding::encode(collection));
let response = client
.get(&url)
.header("Authorization", format!("Bearer {}", config.admin.access_jwt))
.send()
.await?;
if !response.status().is_success() {
return Ok(Vec::new()); // Return empty if collection doesn't exist yet
}
let list_response: serde_json::Value = response.json().await?;
let records = list_response["records"].as_array().unwrap_or(&Vec::new()).clone();
Ok(records)
}
fn extract_slug_from_url(url: &str) -> String {
// Extract slug from URL like "/posts/2025-06-06-ailog.html"
url.split('/')
.last()
.unwrap_or("")
.trim_end_matches(".html")
.to_string()
}
async fn generate_and_store_translation(
client: &reqwest::Client,
config: &AuthConfig,
post: &BlogPost,
ollama_host: &str,
ai_did: &str,
) -> Result<()> {
// Generate translation
let translation = generate_ai_content(&post.title, "translate", ollama_host).await?;
// Store in ai.syui.log.chat.lang collection
let record_data = serde_json::json!({
"post_slug": extract_slug_from_url(&post.href),
"post_title": post.title,
"post_url": post.href,
"lang": "en",
"content": translation,
"generated_at": chrono::Utc::now().to_rfc3339(),
"ai_did": ai_did
});
store_atproto_record(client, config, &config.collections.chat_lang(), &record_data).await
}
async fn generate_and_store_comment(
client: &reqwest::Client,
config: &AuthConfig,
post: &BlogPost,
ollama_host: &str,
ai_did: &str,
) -> Result<()> {
// Generate comment
let comment = generate_ai_content(&post.title, "comment", ollama_host).await?;
// Store in ai.syui.log.chat.comment collection
let record_data = serde_json::json!({
"post_slug": extract_slug_from_url(&post.href),
"post_title": post.title,
"post_url": post.href,
"content": comment,
"generated_at": chrono::Utc::now().to_rfc3339(),
"ai_did": ai_did
});
store_atproto_record(client, config, &config.collections.chat_comment(), &record_data).await
}
async fn store_atproto_record(
client: &reqwest::Client,
config: &AuthConfig,
collection: &str,
record_data: &serde_json::Value,
) -> Result<()> {
let url = format!("{}/xrpc/com.atproto.repo.putRecord", config.admin.pds);
let put_request = serde_json::json!({
"repo": config.admin.did,
"collection": collection,
"rkey": uuid::Uuid::new_v4().to_string(),
"record": record_data
});
let response = client
.post(&url)
.header("Authorization", format!("Bearer {}", config.admin.access_jwt))
.header("Content-Type", "application/json")
.json(&put_request)
.send()
.await?;
if !response.status().is_success() {
let error_text = response.text().await?;
return Err(anyhow::anyhow!("Failed to store record: {}", error_text));
}
Ok(())
}