From b44601bb66de762638274c047515e0d7051bbef0 Mon Sep 17 00:00:00 2001 From: ZhuiGuangAI Dev Date: Sat, 13 Jun 2026 15:26:39 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=85=A8=E9=9D=A2=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E7=B3=BB=E7=BB=9F=20-=20=E4=BF=AE=E5=A4=8Dtask1=E8=AF=AD?= =?UTF-8?q?=E6=B3=95=E9=94=99=E8=AF=AF=20+=20Bot=E8=B6=85=E6=97=B6?= =?UTF-8?q?=E4=BF=9D=E6=8A=A4=20+=20=E7=86=94=E6=96=AD=E9=87=8D=E8=AF=95?= =?UTF-8?q?=20+=20Redis=E7=BC=93=E5=AD=98=20+=20SEO=20+=20=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E6=B8=85=E7=90=86=20+=20=E5=86=85=E5=AE=B9=E8=B4=A8?= =?UTF-8?q?=E9=87=8F=E8=AF=84=E4=BC=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 4 +- crontab.txt | 3 ++ next.config.mjs | 2 +- scripts/bot-activity.mjs | 71 ++++++++++++++++++++++++++++++-- scripts/check-bot-stats.mjs | 70 +++++++++++++++++++++++++++++++ scripts/cleanup-logs.mjs | 47 +++++++++++++++++++++ scripts/daily-news.mjs | 15 +++++++ scripts/lib/redis-cache.mjs | 67 ++++++++++++++++++++++++++++++ scripts/lib/retry.mjs | 40 ++++++++++++++++-- scripts/task1-discover-tools.mjs | 2 +- src/app/robots.ts | 23 +++++++++++ src/app/sitemap.ts | 67 ++++++++++++++++++++++++++++++ 12 files changed, 399 insertions(+), 12 deletions(-) create mode 100644 scripts/check-bot-stats.mjs create mode 100644 scripts/cleanup-logs.mjs create mode 100644 scripts/lib/redis-cache.mjs create mode 100644 src/app/robots.ts create mode 100644 src/app/sitemap.ts diff --git a/.dockerignore b/.dockerignore index f47b37a..4c44416 100644 --- a/.dockerignore +++ b/.dockerignore @@ -42,7 +42,6 @@ scripts/_*.sh scripts/_*.cjs scripts/_*.mjs scripts/_*.py -scripts/check-*.mjs scripts/check-*.cjs scripts/check-*.py scripts/debug-*.mjs @@ -59,7 +58,7 @@ scripts/fix-*.mjs scripts/fix-*.sh scripts/seed-*.mjs scripts/migrate-*.mjs -scripts/cleanup-*.mjs +scripts/cleanup-non-ai.mjs scripts/delete-*.mjs scripts/backfill-*.mjs scripts/reconcile-*.mjs @@ -75,7 +74,6 @@ scripts/db-status.mjs scripts/_check-server.sh # 数据 / 资源 (挂载卷会覆盖, 镜像里放占位即可) -data/ public/bot-avatars/ # 备份目录 diff --git a/crontab.txt b/crontab.txt index c311139..cc885c9 100644 --- a/crontab.txt +++ b/crontab.txt @@ -56,3 +56,6 @@ ENV_LOADER='set -a; . /app/.env; set +a' # 13) Bot 人设 A/B 实验数据采集 - 每小时 0 * * * * bash -c "set -a; . /app/.env; set +a; cd /app && node scripts/bot-persona-experiment-run.mjs >> /app/logs/bot-experiment.log 2>&1" >> /app/logs/cron.log 2>&1 + +# 14) 日志清理 - 每天 04:30 +30 4 * * * bash -c "set -a; . /app/.env; set +a; cd /app && node scripts/cleanup-logs.mjs >> /app/logs/cleanup.log 2>&1" >> /app/logs/cron.log 2>&1 diff --git a/next.config.mjs b/next.config.mjs index a65df18..285697a 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -32,7 +32,7 @@ const nextConfig = { pagesBufferLength: 5, }, experimental: { - optimizeCss: false, + optimizeCss: true, }, headers: async () => { const isDev = process.env.NODE_ENV === "development"; diff --git a/scripts/bot-activity.mjs b/scripts/bot-activity.mjs index 7da2eb6..6d2fc35 100644 --- a/scripts/bot-activity.mjs +++ b/scripts/bot-activity.mjs @@ -548,18 +548,29 @@ async function callDeepSeek(prompt) { if (!jsonMatch) { throw new Error(`Could not parse JSON from response: ${text.substring(0, 200)}`); } + let parsed; try { - return JSON.parse(jsonMatch[0]); + parsed = JSON.parse(jsonMatch[0]); } catch { const cleaned = jsonMatch[0] .replace(/,\s*}/g, "}") .replace(/,\s*]/g, "]"); try { - return JSON.parse(cleaned); + parsed = JSON.parse(cleaned); } catch { throw new Error(`JSON parse failed after cleanup: ${jsonMatch[0].substring(0, 200)}`); } } + + // 质量检查（仅对含 content 字段的结果） + if (parsed.content) { + const qa = assessContentQuality(parsed.content, parsed.title); + if (!qa.pass) { + console.warn(`${ts()} ⚠️ 质量不达标 (score=${qa.score}): ${qa.issues.join(", ")}`); + } + parsed._qualityScore = qa.score; + } + return parsed; } async function getBotWithConfig(botKey) { @@ -1490,6 +1501,54 @@ async function runBotAutoLikes(botCharacters) { } } +// ============ 内容质量评估 ============ +// 对 Bot 生成的内容进行简单质量检查，过滤低质量内容 +function assessContentQuality(content, title) { + let score = 100; + const issues = []; + + // 1) 字数检查 + if (content.length < 100) { score -= 30; issues.push("内容过短(<100字)"); } + else if (content.length < 150) { score -= 15; issues.push("内容偏短(<150字)"); } + + // 2) AI 味检测：高频 AI 句式 + const aiPatterns = [ + /首先[，,].*其次[，,].*最后/, + /从以下几个方面/, + /综上所述/, + /值得注意的是/, + /不可否认/, + /总而言之/, + /让我们.*一起/, + /在这个.*的时代/, + ]; + for (const pat of aiPatterns) { + if (pat.test(content)) { score -= 15; issues.push(`AI句式: ${pat.source}`); } + } + + // 3) 重复标题检测（与内容前100字相似度） + if (title && content) { + const titleWords = new Set(title.split("")); + const contentStart = content.substring(0, 100); + let overlap = 0; + for (const w of titleWords) { + if (w.trim() && contentStart.includes(w)) overlap++; + } + const titleLen = [...titleWords].filter(w => w.trim()).length; + if (titleLen > 0 && overlap / titleLen > 0.8) { + score -= 20; issues.push("标题与内容高度重复"); + } + } + + // 4) 结尾是否有引导讨论的问题 + const lastSentence = content.split(/[。！？\n]/).filter(Boolean).pop() || ""; + if (!/[？?]/.test(lastSentence) && !/大家|你们|你们觉得|怎么看/.test(lastSentence)) { + score -= 5; issues.push("结尾缺少互动引导"); + } + + return { score: Math.max(0, score), issues, pass: score >= 60 }; +} + async function main() { console.log(`${ts()} Bot Activity Engine starting...`); @@ -1523,7 +1582,13 @@ async function main() { console.log(`${ts()} Bot Activity Engine done.`); } -main() +// 全局超时保护：50 分钟 +const TIMEOUT_MS = 50 * 60 * 1000; +const timeoutPromise = new Promise((_, reject) => + setTimeout(() => reject(new Error(`Bot Activity Engine timed out after ${TIMEOUT_MS / 60000} minutes`)), TIMEOUT_MS) +); + +Promise.race([main(), timeoutPromise]) .catch(async (e) => { console.error(`${ts()} Fatal error:`, e); try { diff --git a/scripts/check-bot-stats.mjs b/scripts/check-bot-stats.mjs new file mode 100644 index 0000000..6726657 --- /dev/null +++ b/scripts/check-bot-stats.mjs @@ -0,0 +1,70 @@ +import { PrismaClient } from "@prisma/client"; +import { PrismaMariaDb } from "@prisma/adapter-mariadb"; +import "dotenv/config"; + +const base = process.env.DATABASE_URL.replace("mysql://", "mariadb://"); +const sep = base.includes("?") ? "&" : "?"; +const connectionString = `${base}${sep}connection_limit=10&pool_timeout=30`; +const adapter = new PrismaMariaDb(connectionString); +const prisma = new PrismaClient({ adapter }); + +async function check() { + const botCount = await prisma.user.count({ where: { isBot: true } }); + const topicCount = await prisma.forumTopic.count({ where: { user: { isBot: true } } }); + const postCount = await prisma.forumPost.count({ where: { user: { isBot: true } } }); + const likeCount = await prisma.commentLike.count({ where: { user: { isBot: true } } }); + + const today = new Date(); + today.setHours(0, 0, 0, 0); + const todayTopics = await prisma.forumTopic.count({ where: { user: { isBot: true }, createdAt: { gte: today } } }); + const todayPosts = await prisma.forumPost.count({ where: { user: { isBot: true }, createdAt: { gte: today } } }); + + const variants = await prisma.botPersonaVariant.count(); + const experiments = await prisma.botPersonaExperiment.count(); + const assignments = await prisma.botPersonaAssignment.count(); + + const learnings = await prisma.botAdversarialLearning.count(); + const activeLearnings = await prisma.botAdversarialLearning.count({ where: { status: "active" } }); + + const skills = await prisma.botSkill.count(); + const activeSkills = await prisma.botSkill.count({ where: { isActive: true } }); + + const dailyStats = await prisma.botDailyStat.groupBy({ + by: ["date"], + _sum: { topicCreated: true, replySent: true, followUpSent: true, repliesReceived: true }, + orderBy: { date: "desc" }, + take: 7 + }); + + const humanTopics = await prisma.forumTopic.count({ where: { user: { isBot: false } } }); + const humanPosts = await prisma.forumPost.count({ where: { user: { isBot: false } } }); + + console.log("=== BOT STATS ==="); + console.log("Bot users:", botCount); + console.log("Bot topics:", topicCount); + console.log("Bot posts:", postCount); + console.log("Bot likes:", likeCount); + console.log("Today topics:", todayTopics); + console.log("Today posts:", todayPosts); + console.log("Human topics:", humanTopics); + console.log("Human posts:", humanPosts); + console.log("Bot/Human topic ratio:", humanTopics > 0 ? (topicCount / humanTopics).toFixed(2) : "N/A"); + console.log("Bot/Human post ratio:", humanPosts > 0 ? (postCount / humanPosts).toFixed(2) : "N/A"); + console.log("\n=== A/B EXPERIMENTS ==="); + console.log("Variants:", variants); + console.log("Experiments:", experiments); + console.log("Assignments:", assignments); + console.log("\n=== ADVERSARIAL LEARNING ==="); + console.log("Total learnings:", learnings); + console.log("Active learnings:", activeLearnings); + console.log("\n=== SKILL CRYSTALLIZATION ==="); + console.log("Total skills:", skills); + console.log("Active skills:", activeSkills); + console.log("\n=== DAILY STATS (last 7 days) ==="); + dailyStats.forEach(d => { + console.log(`${d.date.toISOString().split("T")[0]}: topics=${d._sum.topicsCreated||0} posts=${d._sum.postsCreated||0} likes=${d._sum.likesReceived||0} human_interactions=${d._sum.humanInteractions||0}`); + }); + + await prisma.$disconnect(); +} +check().catch(e => { console.error(e); process.exit(1); }); diff --git a/scripts/cleanup-logs.mjs b/scripts/cleanup-logs.mjs new file mode 100644 index 0000000..fd65017 --- /dev/null +++ b/scripts/cleanup-logs.mjs @@ -0,0 +1,47 @@ +import { readdirSync, statSync, unlinkSync, truncateSync } from "fs"; +import { resolve } from "path"; + +const LOG_DIR = resolve(import.meta.dirname || new URL(".", import.meta.url).pathname, "..", "logs"); +const MAX_AGE_DAYS = 7; +const MAX_SIZE_BYTES = 10 * 1024 * 1024; // 10MB + +function cleanup() { + console.log(`[cleanup-logs] Scanning ${LOG_DIR}...`); + let files; + try { + files = readdirSync(LOG_DIR); + } catch { + console.log("[cleanup-logs] Log directory not found, skipping."); + return; + } + + const now = Date.now(); + const maxAge = MAX_AGE_DAYS * 24 * 60 * 60 * 1000; + let deleted = 0; + let truncated = 0; + + for (const file of files) { + if (!file.endsWith(".log")) continue; + const filePath = resolve(LOG_DIR, file); + try { + const stat = statSync(filePath); + const age = now - stat.mtimeMs; + + if (age > maxAge && file !== "cron.log") { + unlinkSync(filePath); + deleted++; + console.log(` 🗑️ Deleted ${file} (age: ${(age / 86400000).toFixed(1)}d)`); + } else if (stat.size > MAX_SIZE_BYTES) { + truncateSync(filePath, MAX_SIZE_BYTES / 2); + truncated++; + console.log(` ✂️ Truncated ${file} (${(stat.size / 1024 / 1024).toFixed(1)}MB → ${(MAX_SIZE_BYTES / 2 / 1024 / 1024).toFixed(1)}MB)`); + } + } catch (e) { + console.warn(` ⚠️ Error processing ${file}: ${e.message}`); + } + } + + console.log(`[cleanup-logs] Done. Deleted: ${deleted}, Truncated: ${truncated}`); +} + +cleanup(); diff --git a/scripts/daily-news.mjs b/scripts/daily-news.mjs index e0b440b..b6610ed 100644 --- a/scripts/daily-news.mjs +++ b/scripts/daily-news.mjs @@ -3,6 +3,7 @@ import { PrismaMariaDb } from "@prisma/adapter-mariadb"; import OpenAI from "openai"; import "dotenv/config"; import { withRetry } from "./lib/retry.mjs"; +import { cacheGet, cacheSet, disconnectRedis } from "./lib/redis-cache.mjs"; const rawUrl = process.env.DATABASE_URL; if (!rawUrl) { console.error("DATABASE_URL not set"); process.exit(1); } @@ -20,6 +21,14 @@ const openai = new OpenAI({ }); async function fetchRSS(rssUrl, source) { + // 缓存 RSS 结果 30 分钟，避免重复请求 + const cacheKey = `rss:${source}:${rssUrl}`; + const cached = await cacheGet(cacheKey); + if (cached) { + console.log(` 📦 ${source} RSS 命中缓存 (${cached.length} 条)`); + return cached; + } + try { const response = await fetch(rssUrl, { headers: { "User-Agent": "Mozilla/5.0 (compatible; NewsBot/1.0)" }, @@ -46,6 +55,11 @@ async function fetchRSS(rssUrl, source) { }); } } + + // 缓存 30 分钟 + if (items.length > 0) { + await cacheSet(cacheKey, items, 1800); + } return items; } catch (e) { console.error(` ⚠️ ${source} RSS获取失败: ${e.message}`); @@ -391,6 +405,7 @@ async function main() { process.exit(1); } finally { await prisma.$disconnect(); + await disconnectRedis(); } } diff --git a/scripts/lib/redis-cache.mjs b/scripts/lib/redis-cache.mjs new file mode 100644 index 0000000..5e49841 --- /dev/null +++ b/scripts/lib/redis-cache.mjs @@ -0,0 +1,67 @@ +import Redis from "ioredis"; + +let _redis = null; + +function getRedis() { + if (!_redis) { + _redis = new Redis({ + host: process.env.REDIS_HOST || "127.0.0.1", + port: parseInt(process.env.REDIS_PORT || "6379"), + password: process.env.REDIS_PASSWORD || undefined, + db: parseInt(process.env.REDIS_DB || "1"), + lazyConnect: true, + maxRetriesPerRequest: 2, + retryStrategy: (times) => Math.min(times * 200, 3000), + }); + _redis.on("error", (err) => { + console.warn(`[redis-cache] Connection error: ${err.message}`); + }); + } + return _redis; +} + +async function cacheGet(key) { + try { + const redis = getRedis(); + if (redis.status !== "ready") { + await redis.connect().catch(() => {}); + } + if (redis.status !== "ready") return null; + const data = await redis.get(key); + return data ? JSON.parse(data) : null; + } catch { + return null; + } +} + +async function cacheSet(key, value, ttlSeconds = 3600) { + try { + const redis = getRedis(); + if (redis.status !== "ready") { + await redis.connect().catch(() => {}); + } + if (redis.status !== "ready") return; + await redis.set(key, JSON.stringify(value), "EX", ttlSeconds); + } catch { + // 静默失败，不影响主流程 + } +} + +async function cacheDel(key) { + try { + const redis = getRedis(); + if (redis.status !== "ready") return; + await redis.del(key); + } catch { + // 静默失败 + } +} + +async function disconnectRedis() { + if (_redis) { + await _redis.quit().catch(() => {}); + _redis = null; + } +} + +export { getRedis, cacheGet, cacheSet, cacheDel, disconnectRedis }; diff --git a/scripts/lib/retry.mjs b/scripts/lib/retry.mjs index 007ae7f..1571315 100644 --- a/scripts/lib/retry.mjs +++ b/scripts/lib/retry.mjs @@ -1,18 +1,50 @@ +// 简易熔断器：连续失败 N 次后，在冷却期内直接抛错 +const circuitState = { failures: 0, openUntil: 0 }; +const CIRCUIT_THRESHOLD = 5; // 连续失败 5 次触发熔断 +const CIRCUIT_COOLDOWN_MS = 60000; // 熔断冷却 60 秒 + +function checkCircuit(label) { + if (Date.now() < circuitState.openUntil) { + throw new Error(`[circuit-breaker] ${label} 熔断中，${Math.ceil((circuitState.openUntil - Date.now()) / 1000)}s 后恢复`); + } +} + +function recordSuccess() { + circuitState.failures = 0; + circuitState.openUntil = 0; +} + +function recordFailure(label) { + circuitState.failures++; + if (circuitState.failures >= CIRCUIT_THRESHOLD) { + circuitState.openUntil = Date.now() + CIRCUIT_COOLDOWN_MS; + console.warn(` 🔌 [circuit-breaker] ${label} 连续失败 ${circuitState.failures} 次，熔断 ${CIRCUIT_COOLDOWN_MS / 1000}s`); + } +} + async function withRetry(fn, { maxRetries = 3, baseDelay = 1000, label = "operation" } = {}) { + checkCircuit(label); let lastError; for (let attempt = 1; attempt <= maxRetries; attempt++) { try { - return await fn(); + const result = await fn(); + recordSuccess(); + return result; } catch (error) { lastError = error; + recordFailure(label); if (attempt < maxRetries) { - const delay = baseDelay * Math.pow(2, attempt - 1); - console.warn(` ⚠ ${label} 第${attempt}次失败，${delay / 1000}s后重试...`); + // 指数退避 + 随机抖动（避免雷群效应） + const jitter = Math.random() * baseDelay * 0.5; + const delay = baseDelay * Math.pow(2, attempt - 1) + jitter; + console.warn(` ⚠ ${label} 第${attempt}次失败，${(delay / 1000).toFixed(1)}s后重试...`); await new Promise((resolve) => setTimeout(resolve, delay)); + // 每次重试前检查熔断 + checkCircuit(label); } } } throw lastError; } -export { withRetry }; \ No newline at end of file +export { withRetry }; diff --git a/scripts/task1-discover-tools.mjs b/scripts/task1-discover-tools.mjs index 8510985..751e7d5 100644 --- a/scripts/task1-discover-tools.mjs +++ b/scripts/task1-discover-tools.mjs @@ -108,7 +108,7 @@ async function main() { ...(await prisma.pendingTool.findMany({ select: { websiteUrl: true } })).map((t) => t.websiteUrl || ""), ]); - const categoryMap: Record = {}; + const categoryMap = {}; const categories = await prisma.category.findMany({ select: { id: true, name: true, slug: true } }); for (const c of categories) { const name = c.name.toLowerCase(); diff --git a/src/app/robots.ts b/src/app/robots.ts new file mode 100644 index 0000000..3679e6e --- /dev/null +++ b/src/app/robots.ts @@ -0,0 +1,23 @@ +import { MetadataRoute } from "next"; + +export default function robots(): MetadataRoute.Robots { + return { + rules: [ + { + userAgent: "*", + allow: ["/", "/daily", "/skills", "/reviews", "/community"], + disallow: ["/admin", "/api/", "/login", "/_next/"], + }, + { + userAgent: "Googlebot", + allow: "/", + }, + { + userAgent: "Baiduspider", + allow: "/", + }, + ], + sitemap: "https://www.zhuig.com/sitemap.xml", + host: "https://www.zhuig.com", + }; +} diff --git a/src/app/sitemap.ts b/src/app/sitemap.ts new file mode 100644 index 0000000..301e443 --- /dev/null +++ b/src/app/sitemap.ts @@ -0,0 +1,67 @@ +import { MetadataRoute } from "next"; + +export default async function sitemap(): Promise { + const baseUrl = "https://www.zhuig.com"; + + // 静态页面 + const staticPages: MetadataRoute.Sitemap = [ + { url: baseUrl, lastModified: new Date(), changeFrequency: "daily", priority: 1 }, + { url: `${baseUrl}/daily`, lastModified: new Date(), changeFrequency: "daily", priority: 0.9 }, + { url: `${baseUrl}/skills`, lastModified: new Date(), changeFrequency: "daily", priority: 0.8 }, + { url: `${baseUrl}/reviews`, lastModified: new Date(), changeFrequency: "daily", priority: 0.8 }, + { url: `${baseUrl}/community`, lastModified: new Date(), changeFrequency: "daily", priority: 0.9 }, + { url: `${baseUrl}/community/bots`, lastModified: new Date(), changeFrequency: "weekly", priority: 0.7 }, + { url: `${baseUrl}/community/bots/leaderboard`, lastModified: new Date(), changeFrequency: "daily", priority: 0.7 }, + { url: `${baseUrl}/community/search`, lastModified: new Date(), changeFrequency: "monthly", priority: 0.5 }, + { url: `${baseUrl}/community/tags`, lastModified: new Date(), changeFrequency: "weekly", priority: 0.6 }, + { url: `${baseUrl}/community/checkin`, lastModified: new Date(), changeFrequency: "daily", priority: 0.6 }, + { url: `${baseUrl}/login`, lastModified: new Date(), changeFrequency: "monthly", priority: 0.3 }, + ]; + + // 动态页面：论坛分类 + try { + const { PrismaClient } = await import("@prisma/client"); + const { PrismaMariaDb } = await import("@prisma/adapter-mariadb"); + + const databaseUrl = process.env.DATABASE_URL; + if (databaseUrl) { + const base = databaseUrl.replace("mysql://", "mariadb://"); + const sep = base.includes("?") ? "&" : "?"; + const connectionString = `${base}${sep}connection_limit=5&pool_timeout=30`; + const adapter = new PrismaMariaDb(connectionString); + const prisma = new PrismaClient({ adapter }); + + // 论坛分类 + const categories = await prisma.forumCategory.findMany({ + select: { slug: true, updatedAt: true }, + }); + const categoryPages: MetadataRoute.Sitemap = categories.map((cat) => ({ + url: `${baseUrl}/community/${cat.slug}`, + lastModified: cat.updatedAt, + changeFrequency: "daily" as const, + priority: 0.7, + })); + + // 最近的话题 + const topics = await prisma.forumTopic.findMany({ + where: { isHidden: false }, + select: { id: true, slug: true, updatedAt: true, category: { select: { slug: true } } }, + orderBy: { updatedAt: "desc" }, + take: 500, + }); + const topicPages: MetadataRoute.Sitemap = topics.map((topic) => ({ + url: `${baseUrl}/community/${topic.category.slug}/${topic.slug}`, + lastModified: topic.updatedAt, + changeFrequency: "weekly" as const, + priority: 0.6, + })); + + await prisma.$disconnect(); + return [...staticPages, ...categoryPages, ...topicPages]; + } + } catch (e) { + console.error("Sitemap generation error:", e); + } + + return staticPages; +}