feat: 全面优化系统 - 修复task1语法错误 + Bot超时保护 + 熔断重试 + Redis缓存 + SEO + 日志清理 + 内容质量评估

.dockerignore

@@ -42,7 +42,6 @@ scripts/_*.sh
 scripts/_*.cjs
 scripts/_*.mjs
 scripts/_*.py
-scripts/check-*.mjs
 scripts/check-*.cjs
 scripts/check-*.py
 scripts/debug-*.mjs
@@ -59,7 +58,7 @@ scripts/fix-*.mjs
 scripts/fix-*.sh
 scripts/seed-*.mjs
 scripts/migrate-*.mjs
-scripts/cleanup-*.mjs
+scripts/cleanup-non-ai.mjs
 scripts/delete-*.mjs
 scripts/backfill-*.mjs
 scripts/reconcile-*.mjs
@@ -75,7 +74,6 @@ scripts/db-status.mjs
 scripts/_check-server.sh
 
 # 数据 / 资源 (挂载卷会覆盖, 镜像里放占位即可)
-data/
 public/bot-avatars/
 
 # 备份目录

crontab.txt

@@ -56,3 +56,6 @@ ENV_LOADER='set -a; . /app/.env; set +a'
 
 # 13) Bot 人设 A/B 实验数据采集 - 每小时
 0 * * * * bash -c "set -a; . /app/.env; set +a; cd /app && node scripts/bot-persona-experiment-run.mjs >> /app/logs/bot-experiment.log 2>&1" >> /app/logs/cron.log 2>&1
+
+# 14) 日志清理 - 每天 04:30
+30 4 * * * bash -c "set -a; . /app/.env; set +a; cd /app && node scripts/cleanup-logs.mjs >> /app/logs/cleanup.log 2>&1" >> /app/logs/cron.log 2>&1

next.config.mjs

@@ -32,7 +32,7 @@ const nextConfig = {
     pagesBufferLength: 5,
   },
   experimental: {
-    optimizeCss: false,
+    optimizeCss: true,
   },
   headers: async () => {
     const isDev = process.env.NODE_ENV === "development";

scripts/bot-activity.mjs

@@ -548,18 +548,29 @@ async function callDeepSeek(prompt) {
   if (!jsonMatch) {
     throw new Error(`Could not parse JSON from response: ${text.substring(0, 200)}`);
   }
+  let parsed;
   try {
-    return JSON.parse(jsonMatch[0]);
+    parsed = JSON.parse(jsonMatch[0]);
   } catch {
     const cleaned = jsonMatch[0]
       .replace(/,\s*}/g, "}")
       .replace(/,\s*]/g, "]");
     try {
-      return JSON.parse(cleaned);
+      parsed = JSON.parse(cleaned);
     } catch {
       throw new Error(`JSON parse failed after cleanup: ${jsonMatch[0].substring(0, 200)}`);
     }
   }
+
+  // 质量检查（仅对含 content 字段的结果）
+  if (parsed.content) {
+    const qa = assessContentQuality(parsed.content, parsed.title);
+    if (!qa.pass) {
+      console.warn(`${ts()}   ⚠️ 质量不达标 (score=${qa.score}): ${qa.issues.join(", ")}`);
+    }
+    parsed._qualityScore = qa.score;
+  }
+  return parsed;
 }
 
 async function getBotWithConfig(botKey) {
@@ -1490,6 +1501,54 @@ async function runBotAutoLikes(botCharacters) {
   }
 }
 
+// ============ 内容质量评估 ============
+// 对 Bot 生成的内容进行简单质量检查，过滤低质量内容
+function assessContentQuality(content, title) {
+  let score = 100;
+  const issues = [];
+
+  // 1) 字数检查
+  if (content.length < 100) { score -= 30; issues.push("内容过短(<100字)"); }
+  else if (content.length < 150) { score -= 15; issues.push("内容偏短(<150字)"); }
+
+  // 2) AI 味检测：高频 AI 句式
+  const aiPatterns = [
+    /首先[，,].*其次[，,].*最后/,
+    /从以下几个方面/,
+    /综上所述/,
+    /值得注意的是/,
+    /不可否认/,
+    /总而言之/,
+    /让我们.*一起/,
+    /在这个.*的时代/,
+  ];
+  for (const pat of aiPatterns) {
+    if (pat.test(content)) { score -= 15; issues.push(`AI句式: ${pat.source}`); }
+  }
+
+  // 3) 重复标题检测（与内容前100字相似度）
+  if (title && content) {
+    const titleWords = new Set(title.split(""));
+    const contentStart = content.substring(0, 100);
+    let overlap = 0;
+    for (const w of titleWords) {
+      if (w.trim() && contentStart.includes(w)) overlap++;
+    }
+    const titleLen = [...titleWords].filter(w => w.trim()).length;
+    if (titleLen > 0 && overlap / titleLen > 0.8) {
+      score -= 20; issues.push("标题与内容高度重复");
+    }
+  }
+
+  // 4) 结尾是否有引导讨论的问题
+  const lastSentence = content.split(/[。！？\n]/).filter(Boolean).pop() || "";
+  if (!/[？?]/.test(lastSentence) && !/大家|你们|你们觉得|怎么看/.test(lastSentence)) {
+    score -= 5; issues.push("结尾缺少互动引导");
+  }
+
+  return { score: Math.max(0, score), issues, pass: score >= 60 };
+}
+
 async function main() {
   console.log(`${ts()} Bot Activity Engine starting...`);
 
@@ -1523,7 +1582,13 @@ async function main() {
   console.log(`${ts()} Bot Activity Engine done.`);
 }
 
-main()
+// 全局超时保护：50 分钟
+const TIMEOUT_MS = 50 * 60 * 1000;
+const timeoutPromise = new Promise((_, reject) =>
+  setTimeout(() => reject(new Error(`Bot Activity Engine timed out after ${TIMEOUT_MS / 60000} minutes`)), TIMEOUT_MS)
+);
+
+Promise.race([main(), timeoutPromise])
   .catch(async (e) => {
     console.error(`${ts()} Fatal error:`, e);
     try {

scripts/check-bot-stats.mjs

@@ -0,0 +1,70 @@
+import { PrismaClient } from "@prisma/client";
+import { PrismaMariaDb } from "@prisma/adapter-mariadb";
+import "dotenv/config";
+
+const base = process.env.DATABASE_URL.replace("mysql://", "mariadb://");
+const sep = base.includes("?") ? "&" : "?";
+const connectionString = `${base}${sep}connection_limit=10&pool_timeout=30`;
+const adapter = new PrismaMariaDb(connectionString);
+const prisma = new PrismaClient({ adapter });
+
+async function check() {
+  const botCount = await prisma.user.count({ where: { isBot: true } });
+  const topicCount = await prisma.forumTopic.count({ where: { user: { isBot: true } } });
+  const postCount = await prisma.forumPost.count({ where: { user: { isBot: true } } });
+  const likeCount = await prisma.commentLike.count({ where: { user: { isBot: true } } });
+
+  const today = new Date();
+  today.setHours(0, 0, 0, 0);
+  const todayTopics = await prisma.forumTopic.count({ where: { user: { isBot: true }, createdAt: { gte: today } } });
+  const todayPosts = await prisma.forumPost.count({ where: { user: { isBot: true }, createdAt: { gte: today } } });
+
+  const variants = await prisma.botPersonaVariant.count();
+  const experiments = await prisma.botPersonaExperiment.count();
+  const assignments = await prisma.botPersonaAssignment.count();
+
+  const learnings = await prisma.botAdversarialLearning.count();
+  const activeLearnings = await prisma.botAdversarialLearning.count({ where: { status: "active" } });
+
+  const skills = await prisma.botSkill.count();
+  const activeSkills = await prisma.botSkill.count({ where: { isActive: true } });
+
+  const dailyStats = await prisma.botDailyStat.groupBy({
+    by: ["date"],
+    _sum: { topicCreated: true, replySent: true, followUpSent: true, repliesReceived: true },
+    orderBy: { date: "desc" },
+    take: 7
+  });
+
+  const humanTopics = await prisma.forumTopic.count({ where: { user: { isBot: false } } });
+  const humanPosts = await prisma.forumPost.count({ where: { user: { isBot: false } } });
+
+  console.log("=== BOT STATS ===");
+  console.log("Bot users:", botCount);
+  console.log("Bot topics:", topicCount);
+  console.log("Bot posts:", postCount);
+  console.log("Bot likes:", likeCount);
+  console.log("Today topics:", todayTopics);
+  console.log("Today posts:", todayPosts);
+  console.log("Human topics:", humanTopics);
+  console.log("Human posts:", humanPosts);
+  console.log("Bot/Human topic ratio:", humanTopics > 0 ? (topicCount / humanTopics).toFixed(2) : "N/A");
+  console.log("Bot/Human post ratio:", humanPosts > 0 ? (postCount / humanPosts).toFixed(2) : "N/A");
+  console.log("\n=== A/B EXPERIMENTS ===");
+  console.log("Variants:", variants);
+  console.log("Experiments:", experiments);
+  console.log("Assignments:", assignments);
+  console.log("\n=== ADVERSARIAL LEARNING ===");
+  console.log("Total learnings:", learnings);
+  console.log("Active learnings:", activeLearnings);
+  console.log("\n=== SKILL CRYSTALLIZATION ===");
+  console.log("Total skills:", skills);
+  console.log("Active skills:", activeSkills);
+  console.log("\n=== DAILY STATS (last 7 days) ===");
+  dailyStats.forEach(d => {
+    console.log(`${d.date.toISOString().split("T")[0]}: topics=${d._sum.topicsCreated||0} posts=${d._sum.postsCreated||0} likes=${d._sum.likesReceived||0} human_interactions=${d._sum.humanInteractions||0}`);
+  });
+
+  await prisma.$disconnect();
+}
+check().catch(e => { console.error(e); process.exit(1); });

scripts/cleanup-logs.mjs

@@ -0,0 +1,47 @@
+import { readdirSync, statSync, unlinkSync, truncateSync } from "fs";
+import { resolve } from "path";
+
+const LOG_DIR = resolve(import.meta.dirname || new URL(".", import.meta.url).pathname, "..", "logs");
+const MAX_AGE_DAYS = 7;
+const MAX_SIZE_BYTES = 10 * 1024 * 1024; // 10MB
+
+function cleanup() {
+  console.log(`[cleanup-logs] Scanning ${LOG_DIR}...`);
+  let files;
+  try {
+    files = readdirSync(LOG_DIR);
+  } catch {
+    console.log("[cleanup-logs] Log directory not found, skipping.");
+    return;
+  }
+
+  const now = Date.now();
+  const maxAge = MAX_AGE_DAYS * 24 * 60 * 60 * 1000;
+  let deleted = 0;
+  let truncated = 0;
+
+  for (const file of files) {
+    if (!file.endsWith(".log")) continue;
+    const filePath = resolve(LOG_DIR, file);
+    try {
+      const stat = statSync(filePath);
+      const age = now - stat.mtimeMs;
+
+      if (age > maxAge && file !== "cron.log") {
+        unlinkSync(filePath);
+        deleted++;
+        console.log(`  🗑️  Deleted ${file} (age: ${(age / 86400000).toFixed(1)}d)`);
+      } else if (stat.size > MAX_SIZE_BYTES) {
+        truncateSync(filePath, MAX_SIZE_BYTES / 2);
+        truncated++;
+        console.log(`  ✂️  Truncated ${file} (${(stat.size / 1024 / 1024).toFixed(1)}MB → ${(MAX_SIZE_BYTES / 2 / 1024 / 1024).toFixed(1)}MB)`);
+      }
+    } catch (e) {
+      console.warn(`  ⚠️  Error processing ${file}: ${e.message}`);
+    }
+  }
+
+  console.log(`[cleanup-logs] Done. Deleted: ${deleted}, Truncated: ${truncated}`);
+}
+
+cleanup();

scripts/daily-news.mjs

@@ -3,6 +3,7 @@ import { PrismaMariaDb } from "@prisma/adapter-mariadb";
 import OpenAI from "openai";
 import "dotenv/config";
 import { withRetry } from "./lib/retry.mjs";
+import { cacheGet, cacheSet, disconnectRedis } from "./lib/redis-cache.mjs";
 
 const rawUrl = process.env.DATABASE_URL;
 if (!rawUrl) { console.error("DATABASE_URL not set"); process.exit(1); }
@@ -20,6 +21,14 @@ const openai = new OpenAI({
 });
 
 async function fetchRSS(rssUrl, source) {
+  // 缓存 RSS 结果 30 分钟，避免重复请求
+  const cacheKey = `rss:${source}:${rssUrl}`;
+  const cached = await cacheGet(cacheKey);
+  if (cached) {
+    console.log(`   📦 ${source} RSS 命中缓存 (${cached.length} 条)`);
+    return cached;
+  }
+
   try {
     const response = await fetch(rssUrl, {
       headers: { "User-Agent": "Mozilla/5.0 (compatible; NewsBot/1.0)" },
@@ -46,6 +55,11 @@ async function fetchRSS(rssUrl, source) {
         });
       }
     }
+
+    // 缓存 30 分钟
+    if (items.length > 0) {
+      await cacheSet(cacheKey, items, 1800);
+    }
     return items;
   } catch (e) {
     console.error(`   ⚠️ ${source} RSS获取失败: ${e.message}`);
@@ -391,6 +405,7 @@ async function main() {
     process.exit(1);
   } finally {
     await prisma.$disconnect();
+    await disconnectRedis();
   }
 }
 

scripts/lib/redis-cache.mjs

@@ -0,0 +1,67 @@
+import Redis from "ioredis";
+
+let _redis = null;
+
+function getRedis() {
+  if (!_redis) {
+    _redis = new Redis({
+      host: process.env.REDIS_HOST || "127.0.0.1",
+      port: parseInt(process.env.REDIS_PORT || "6379"),
+      password: process.env.REDIS_PASSWORD || undefined,
+      db: parseInt(process.env.REDIS_DB || "1"),
+      lazyConnect: true,
+      maxRetriesPerRequest: 2,
+      retryStrategy: (times) => Math.min(times * 200, 3000),
+    });
+    _redis.on("error", (err) => {
+      console.warn(`[redis-cache] Connection error: ${err.message}`);
+    });
+  }
+  return _redis;
+}
+
+async function cacheGet(key) {
+  try {
+    const redis = getRedis();
+    if (redis.status !== "ready") {
+      await redis.connect().catch(() => {});
+    }
+    if (redis.status !== "ready") return null;
+    const data = await redis.get(key);
+    return data ? JSON.parse(data) : null;
+  } catch {
+    return null;
+  }
+}
+
+async function cacheSet(key, value, ttlSeconds = 3600) {
+  try {
+    const redis = getRedis();
+    if (redis.status !== "ready") {
+      await redis.connect().catch(() => {});
+    }
+    if (redis.status !== "ready") return;
+    await redis.set(key, JSON.stringify(value), "EX", ttlSeconds);
+  } catch {
+    // 静默失败，不影响主流程
+  }
+}
+
+async function cacheDel(key) {
+  try {
+    const redis = getRedis();
+    if (redis.status !== "ready") return;
+    await redis.del(key);
+  } catch {
+    // 静默失败
+  }
+}
+
+async function disconnectRedis() {
+  if (_redis) {
+    await _redis.quit().catch(() => {});
+    _redis = null;
+  }
+}
+
+export { getRedis, cacheGet, cacheSet, cacheDel, disconnectRedis };

scripts/lib/retry.mjs

@@ -1,18 +1,50 @@
+// 简易熔断器：连续失败 N 次后，在冷却期内直接抛错
+const circuitState = { failures: 0, openUntil: 0 };
+const CIRCUIT_THRESHOLD = 5;      // 连续失败 5 次触发熔断
+const CIRCUIT_COOLDOWN_MS = 60000; // 熔断冷却 60 秒
+
+function checkCircuit(label) {
+  if (Date.now() < circuitState.openUntil) {
+    throw new Error(`[circuit-breaker] ${label} 熔断中，${Math.ceil((circuitState.openUntil - Date.now()) / 1000)}s 后恢复`);
+  }
+}
+
+function recordSuccess() {
+  circuitState.failures = 0;
+  circuitState.openUntil = 0;
+}
+
+function recordFailure(label) {
+  circuitState.failures++;
+  if (circuitState.failures >= CIRCUIT_THRESHOLD) {
+    circuitState.openUntil = Date.now() + CIRCUIT_COOLDOWN_MS;
+    console.warn(`  🔌 [circuit-breaker] ${label} 连续失败 ${circuitState.failures} 次，熔断 ${CIRCUIT_COOLDOWN_MS / 1000}s`);
+  }
+}
+
 async function withRetry(fn, { maxRetries = 3, baseDelay = 1000, label = "operation" } = {}) {
+  checkCircuit(label);
   let lastError;
   for (let attempt = 1; attempt <= maxRetries; attempt++) {
     try {
-      return await fn();
+      const result = await fn();
+      recordSuccess();
+      return result;
     } catch (error) {
       lastError = error;
+      recordFailure(label);
       if (attempt < maxRetries) {
-        const delay = baseDelay * Math.pow(2, attempt - 1);
-        console.warn(`  ⚠ ${label} 第${attempt}次失败，${delay / 1000}s后重试...`);
+        // 指数退避 + 随机抖动（避免雷群效应）
+        const jitter = Math.random() * baseDelay * 0.5;
+        const delay = baseDelay * Math.pow(2, attempt - 1) + jitter;
+        console.warn(`  ⚠ ${label} 第${attempt}次失败，${(delay / 1000).toFixed(1)}s后重试...`);
         await new Promise((resolve) => setTimeout(resolve, delay));
+        // 每次重试前检查熔断
+        checkCircuit(label);
       }
     }
   }
   throw lastError;
 }
 
-export { withRetry };
+export { withRetry };

scripts/task1-discover-tools.mjs

@@ -108,7 +108,7 @@ async function main() {
     ...(await prisma.pendingTool.findMany({ select: { websiteUrl: true } })).map((t) => t.websiteUrl || ""),
   ]);
 
-  const categoryMap: Record<string, number> = {};
+  const categoryMap = {};
   const categories = await prisma.category.findMany({ select: { id: true, name: true, slug: true } });
   for (const c of categories) {
     const name = c.name.toLowerCase();

src/app/robots.ts

@@ -0,0 +1,23 @@
+import { MetadataRoute } from "next";
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: [
+      {
+        userAgent: "*",
+        allow: ["/", "/daily", "/skills", "/reviews", "/community"],
+        disallow: ["/admin", "/api/", "/login", "/_next/"],
+      },
+      {
+        userAgent: "Googlebot",
+        allow: "/",
+      },
+      {
+        userAgent: "Baiduspider",
+        allow: "/",
+      },
+    ],
+    sitemap: "https://www.zhuig.com/sitemap.xml",
+    host: "https://www.zhuig.com",
+  };
+}

src/app/sitemap.ts

@@ -0,0 +1,67 @@
+import { MetadataRoute } from "next";
+
+export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
+  const baseUrl = "https://www.zhuig.com";
+
+  // 静态页面
+  const staticPages: MetadataRoute.Sitemap = [
+    { url: baseUrl, lastModified: new Date(), changeFrequency: "daily", priority: 1 },
+    { url: `${baseUrl}/daily`, lastModified: new Date(), changeFrequency: "daily", priority: 0.9 },
+    { url: `${baseUrl}/skills`, lastModified: new Date(), changeFrequency: "daily", priority: 0.8 },
+    { url: `${baseUrl}/reviews`, lastModified: new Date(), changeFrequency: "daily", priority: 0.8 },
+    { url: `${baseUrl}/community`, lastModified: new Date(), changeFrequency: "daily", priority: 0.9 },
+    { url: `${baseUrl}/community/bots`, lastModified: new Date(), changeFrequency: "weekly", priority: 0.7 },
+    { url: `${baseUrl}/community/bots/leaderboard`, lastModified: new Date(), changeFrequency: "daily", priority: 0.7 },
+    { url: `${baseUrl}/community/search`, lastModified: new Date(), changeFrequency: "monthly", priority: 0.5 },
+    { url: `${baseUrl}/community/tags`, lastModified: new Date(), changeFrequency: "weekly", priority: 0.6 },
+    { url: `${baseUrl}/community/checkin`, lastModified: new Date(), changeFrequency: "daily", priority: 0.6 },
+    { url: `${baseUrl}/login`, lastModified: new Date(), changeFrequency: "monthly", priority: 0.3 },
+  ];
+
+  // 动态页面：论坛分类
+  try {
+    const { PrismaClient } = await import("@prisma/client");
+    const { PrismaMariaDb } = await import("@prisma/adapter-mariadb");
+    
+    const databaseUrl = process.env.DATABASE_URL;
+    if (databaseUrl) {
+      const base = databaseUrl.replace("mysql://", "mariadb://");
+      const sep = base.includes("?") ? "&" : "?";
+      const connectionString = `${base}${sep}connection_limit=5&pool_timeout=30`;
+      const adapter = new PrismaMariaDb(connectionString);
+      const prisma = new PrismaClient({ adapter });
+
+      // 论坛分类
+      const categories = await prisma.forumCategory.findMany({
+        select: { slug: true, updatedAt: true },
+      });
+      const categoryPages: MetadataRoute.Sitemap = categories.map((cat) => ({
+        url: `${baseUrl}/community/${cat.slug}`,
+        lastModified: cat.updatedAt,
+        changeFrequency: "daily" as const,
+        priority: 0.7,
+      }));
+
+      // 最近的话题
+      const topics = await prisma.forumTopic.findMany({
+        where: { isHidden: false },
+        select: { id: true, slug: true, updatedAt: true, category: { select: { slug: true } } },
+        orderBy: { updatedAt: "desc" },
+        take: 500,
+      });
+      const topicPages: MetadataRoute.Sitemap = topics.map((topic) => ({
+        url: `${baseUrl}/community/${topic.category.slug}/${topic.slug}`,
+        lastModified: topic.updatedAt,
+        changeFrequency: "weekly" as const,
+        priority: 0.6,
+      }));
+
+      await prisma.$disconnect();
+      return [...staticPages, ...categoryPages, ...topicPages];
+    }
+  } catch (e) {
+    console.error("Sitemap generation error:", e);
+  }
+
+  return staticPages;
+}