全项目扫描修复: Docker数据卷修复+安全requireAdmin+12页SEO+API白名单+脚本超时+常量提取+假数据删除

2026-06-12 17:27:47 +08:00
parent 464924aea0
commit 4c325c8699
454 changed files with 33647 additions and 980 deletions
@@ -0,0 +1,535 @@
+// Bot 数字人 vs 真人"对抗学习" 共享模块
+// 每周为每个 bot 选 1 个真实高赞真人帖，让 bot 吸收其"为什么高互动"的洞察
+//
+// 用法：
+//   import { runWeeklyAdversarialLearning, getActiveLearningForBot, buildAdversarialBlock } from "./lib/bot-adversarial-learning.mjs";
+//
+//   // 1) 周调度入口
+//   await runWeeklyAdversarialLearning({ weekKey: "2026-W23", lookbackDays: 7 });
+//
+//   // 2) 发帖 prompt 注入对抗学习参考
+//   const learning = await getActiveLearningForBot(botConfigId);
+//   const block = buildAdversarialBlock(learning); // → "[对抗学习] ..."
+
+import { PrismaClient } from "@prisma/client";
+import { PrismaMariaDb } from "@prisma/adapter-mariadb";
+import "dotenv/config";
+import OpenAI from "openai";
+import { withRetry } from "./retry.mjs";
+
+const base = (process.env.DATABASE_URL || "").replace("mysql://", "mariadb://");
+const sep = base.includes("?") ? "&" : "?";
+const connectionString = `${base}${sep}connection_limit=3&pool_timeout=10`;
+
+let _prisma = null;
+function getPrisma() {
+  if (!_prisma) {
+    const adapter = new PrismaMariaDb(connectionString);
+    _prisma = new PrismaClient({ adapter });
+  }
+  return _prisma;
+}
+
+let _openai = null;
+function getOpenAI() {
+  if (_openai) return _openai;
+  if (!process.env.DEEPSEEK_API_KEY) return null;
+  _openai = new OpenAI({
+    apiKey: process.env.DEEPSEEK_API_KEY,
+    baseURL: "https://api.deepseek.com/v1",
+  });
+  return _openai;
+}
+
+const MODEL = process.env.DEEPSEEK_MODEL || "deepseek-chat";
+
+function ts() {
+  return `[${new Date().toISOString()}]`;
+}
+
+// ======== 配置 ========
+export const LEARNING_CONFIG = {
+  // 候选帖最小门槛
+  minReplies: 3,
+  minLikes: 5,
+  minHumanReplies: 1,
+  // 回看窗口（天）
+  lookbackDays: 7,
+  // 候选数上限
+  candidateLimit: 30,
+  // 每个 bot 每周上限
+  perBotPerWeek: 1,
+  // 关联度评分
+  sameForumWeight: 1.0,
+  keywordOverlapWeight: 0.6,
+  recencyWeight: 0.3,
+  // 过期
+  expireDays: 7,
+  // LLM 提示 token 上限
+  contentTruncate: 1500,
+};
+
+// ======== 工具：周 key (ISO week) ========
+/**
+ * 形如 "2026-W23"，与 date-fns / Excel WEEKNUM 行为一致
+ */
+export function getISOWeekKey(date = new Date()) {
+  const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate()));
+  const dayNum = d.getUTCDay() || 7; // Mon=1..Sun=7
+  d.setUTCDate(d.getUTCDate() + 4 - dayNum);
+  const yearStart = new Date(Date.UTC(d.getUTCFullYear(), 0, 1));
+  const weekNo = Math.ceil(((d - yearStart) / 86400000 + 1) / 7);
+  return `${d.getUTCFullYear()}-W${String(weekNo).padStart(2, "0")}`;
+}
+
+// ======== 工具：取候选真人帖 ========
+/**
+ * 拉过去 N 天内由真人发布、且互动数超过门槛的 topic 列表
+ * 排序：综合分 = 赞×3 + 真人回复×5 + 回复
+ * 仅返回 topic 维度（post 维度后续可加）
+ */
+export async function fetchHumanHighEngagementTopics(lookbackDays = 7, limit = 30) {
+  const prisma = getPrisma();
+  const since = new Date(Date.now() - lookbackDays * 24 * 60 * 60 * 1000);
+
+  // 1) 先拉 botUserId 集合，避免 N+1
+  const botUsers = await prisma.user.findMany({
+    where: { isBot: true },
+    select: { id: true },
+  });
+  const botUserIdSet = new Set(botUsers.map((u) => u.id));
+
+  // 2) 拉所有非 bot 的 topic，按 replyCount desc 取前 N（粗筛）
+  const candidates = await prisma.forumTopic.findMany({
+    where: {
+      userId: { notIn: Array.from(botUserIdSet) },
+      isHidden: false,
+      createdAt: { gte: since },
+    },
+    include: {
+      user: { select: { id: true, name: true, isBot: true } },
+      category: { select: { slug: true, name: true } },
+      posts: {
+        where: { user: { isBot: false } },
+        select: { id: true },
+      },
+    },
+    orderBy: [{ replyCount: "desc" }, { likeCount: "desc" }],
+    take: limit * 2,
+  });
+
+  // 3) 计算综合分，过门槛
+  const scored = candidates
+    .filter((t) => !t.user?.isBot)
+    .map((t) => {
+      const humanReplies = t.posts?.length || 0;
+      const score = (t.likeCount || 0) * 3 + humanReplies * 5 + (t.replyCount || 0);
+      return { ...t, _score: score, _humanReplies: humanReplies };
+    })
+    .filter(
+      (t) =>
+        (t.replyCount || 0) >= LEARNING_CONFIG.minReplies ||
+        t._humanReplies >= LEARNING_CONFIG.minHumanReplies ||
+        (t.likeCount || 0) >= LEARNING_CONFIG.minLikes
+    )
+    .sort((a, b) => b._score - a._score)
+    .slice(0, limit);
+
+  return scored;
+}
+
+// ======== 工具：候选与 bot 关联度评分 ========
+/**
+ * 计算一篇候选帖对某 bot 的关联度
+ * - 板块命中：bot.primaryForums 包含该帖板块 → 同板块加权
+ * - 关键词重叠：bot stanceKeywords 与帖子标题/内容重叠数
+ * - 时效衰减：越新越好
+ */
+function calcRelevance(bot, persona, topic) {
+  let score = 0;
+  const primaryForums = (bot.primaryForums || []).map((s) => String(s).toLowerCase());
+  const stanceKeywords = ((persona?.stanceKeywords || []).map((k) => k.key || k)).filter(Boolean);
+
+  if (primaryForums.includes(String(topic.category?.slug || "").toLowerCase())) {
+    score += LEARNING_CONFIG.sameForumWeight;
+  }
+
+  if (stanceKeywords.length > 0) {
+    const text = `${topic.title || ""} ${topic.content || ""}`.toLowerCase();
+    let hits = 0;
+    for (const kw of stanceKeywords) {
+      const k = String(kw).toLowerCase();
+      if (k.length < 2) continue;
+      if (text.includes(k)) hits++;
+    }
+    const overlapRatio = Math.min(1, hits / 5);
+    score += overlapRatio * LEARNING_CONFIG.keywordOverlapWeight;
+  }
+
+  // 时效：距今 < 3 天满分，> 6 天 0 分
+  const ageMs = Date.now() - new Date(topic.createdAt).getTime();
+  const ageDays = ageMs / (24 * 60 * 60 * 1000);
+  const recencyScore = Math.max(0, 1 - ageDays / 6) * LEARNING_CONFIG.recencyWeight;
+  score += recencyScore;
+
+  return Math.round(score * 1000) / 1000;
+}
+
+// ======== LLM 提取"为什么高互动"洞察 ========
+const LEARN_PROMPT = (botName, botRole, topic) => `你是【${botName}】的内容策略师，每周要研究一篇真实用户（不是数字人）的高互动帖子，提炼出"为什么火"，作为下周发帖可借鉴的方向。
+
+[数字人角色]
+- 名字：${botName}
+- 角色：${botRole || "通用论坛内容创作者"}
+
+[本周高互动真人帖]
+- 标题：${topic.title}
+- 板块：${topic.category?.name || ""}
+- 收到回复：${topic.replyCount}（真人 ${topic._humanReplies || 0}）
+- 收到点赞：${topic.likeCount}
+- 浏览数：${topic.viewCount}
+- 内容（截取前 ${LEARNING_CONFIG.contentTruncate} 字）：
+${(topic.content || "").slice(0, LEARNING_CONFIG.contentTruncate)}
+
+[任务]
+站在【${botName}】的视角，分析这篇真人帖之所以高互动的 2-3 个可学习点。注意：
+1. 不要泛泛而谈"写得好"，要给出具体的"可复用的写法/角度/钩子"
+2. 关注"为什么能引发真人回复"——是观点鲜明、抛问题、还是给方案/故事
+3. 提炼出 2-3 个对【${botName}】下周发帖有直接借鉴价值的策略
+4. 输出一段 150-250 字的简洁洞察，下周发帖时可作为参考
+
+[输出格式]
+只输出严格的 JSON：
+{
+  "insight": "（150-250 字的洞察，聚焦可复用的写法/角度/钩子）",
+  "categories": ["hook|story|data|question|contrarian|empathy|practical", ...]
+}`;
+
+async function extractInsightWithLLM(botChar, topic) {
+  const openai = getOpenAI();
+  if (!openai) {
+    // 降级：无 LLM 时输出规则性总结
+    return buildFallbackInsight(topic);
+  }
+  const p = botChar?.personality || {};
+  const role = [p.identity, p.stance, p.speakingStyle].filter(Boolean).join(" / ");
+  const prompt = LEARN_PROMPT(botChar?.displayName || "数字人", role, topic);
+  const response = await withRetry(() =>
+    openai.chat.completions.create({
+      model: MODEL,
+      messages: [{ role: "user", content: prompt }],
+      temperature: 0.6,
+      max_tokens: 800,
+    })
+  );
+  const text = response.choices?.[0]?.message?.content?.trim() || "";
+  const match = text.match(/\{[\s\S]*\}/);
+  if (!match) return buildFallbackInsight(topic);
+  try {
+    const parsed = JSON.parse(match[0]);
+    return {
+      insight: String(parsed.insight || "").slice(0, 1000),
+      categories: Array.isArray(parsed.categories) ? parsed.categories.slice(0, 5) : [],
+    };
+  } catch {
+    return buildFallbackInsight(topic);
+  }
+}
+
+function buildFallbackInsight(topic) {
+  return {
+    insight: `这篇真人帖（${topic.title || "无标题"}）收到 ${topic.replyCount} 条回复 / ${topic.likeCount} 个赞，关键在于：${(topic.content || "").slice(0, 80)}...的可复用角度。下周可参考其切入点和表达方式。`,
+    categories: ["practical"],
+  };
+}
+
+// ======== 单 bot 学习流程 ========
+/**
+ * 对一个 bot 跑对抗学习：
+ *  1) 拉真人高互动候选
+ *  2) 与 bot 关联度排序 → 选 top1
+ *  3) LLM 提取洞察
+ *  4) upsert 到 bot_adversarial_learnings (unique: botId+weekKey)
+ *  5) 同步写一条 BotMemory
+ */
+export async function learnForBot(botUser, botConfig, persona, options = {}) {
+  const prisma = getPrisma();
+  const cfg = { ...LEARNING_CONFIG, ...options };
+  const weekKey = options.weekKey || getISOWeekKey();
+
+  // 已存在本 bot 本周的学习 → 跳过
+  const existing = await prisma.botAdversarialLearning.findUnique({
+    where: { botId_weekKey: { botId: botConfig.id, weekKey } },
+  });
+  if (existing) {
+    return { skipped: true, reason: "already_learned", record: existing };
+  }
+
+  // 1) 候选
+  const candidates = await fetchHumanHighEngagementTopics(cfg.lookbackDays, cfg.candidateLimit);
+  if (candidates.length === 0) {
+    return { skipped: true, reason: "no_candidates" };
+  }
+
+  // 2) 关联度排序
+  const ranked = candidates
+    .map((c) => ({ topic: c, relevance: calcRelevance(botConfig, persona, c) }))
+    .sort((a, b) => b.relevance - a.relevance);
+  const top = ranked[0];
+
+  // 3) LLM 提取
+  const { insight, categories } = await extractInsightWithLLM(botUser._botChar, top.topic);
+
+  // 4) upsert 写入
+  const expiresAt = new Date(Date.now() + cfg.expireDays * 24 * 60 * 60 * 1000);
+  const record = await prisma.botAdversarialLearning.upsert({
+    where: { botId_weekKey: { botId: botConfig.id, weekKey } },
+    create: {
+      botId: botConfig.id,
+      weekKey,
+      sourceRefType: "topic",
+      sourceRefId: top.topic.id,
+      sourceUserId: top.topic.userId,
+      sourceUserName: top.topic.user?.name || null,
+      forumSlug: top.topic.category?.slug || null,
+      sourceTitle: top.topic.title || null,
+      sourceContent: (top.topic.content || "").slice(0, 5000),
+      sourceMetrics: {
+        replyCount: top.topic.replyCount || 0,
+        likeCount: top.topic.likeCount || 0,
+        viewCount: top.topic.viewCount || 0,
+        humanReplies: top._humanReplies || 0,
+        engagementScore: top._score,
+      },
+      relevanceScore: top.relevance,
+      learnedInsight: insight,
+      learnCategories: categories,
+      status: "active",
+      expiresAt,
+    },
+    update: {
+      // 本周二次跑：覆盖（强制重学）
+      sourceRefType: "topic",
+      sourceRefId: top.topic.id,
+      sourceUserId: top.topic.userId,
+      sourceUserName: top.topic.user?.name || null,
+      forumSlug: top.topic.category?.slug || null,
+      sourceTitle: top.topic.title || null,
+      sourceContent: (top.topic.content || "").slice(0, 5000),
+      sourceMetrics: {
+        replyCount: top.topic.replyCount || 0,
+        likeCount: top.topic.likeCount || 0,
+        viewCount: top.topic.viewCount || 0,
+        humanReplies: top._humanReplies || 0,
+        engagementScore: top._score,
+      },
+      relevanceScore: top.relevance,
+      learnedInsight: insight,
+      learnCategories: categories,
+      status: "active",
+      expiresAt,
+    },
+  });
+
+  // 5) 同步写一条 BotMemory
+  await prisma.botMemory.create({
+    data: {
+      botId: botConfig.id,
+      memoryType: "adversarial_learning",
+      layer: "LONGTERM",
+      content: {
+        action: "learn_from_human",
+        weekKey,
+        learningId: record.id,
+        sourceTitle: top.topic.title,
+        sourceUserName: top.topic.user?.name || null,
+        sourceUserId: top.topic.userId,
+        sourceRefType: "topic",
+        sourceRefId: top.topic.id,
+        insight,
+        categories,
+        relevanceScore: top.relevance,
+        sourceMetrics: {
+          replyCount: top.topic.replyCount || 0,
+          likeCount: top.topic.likeCount || 0,
+          humanReplies: top._humanReplies || 0,
+        },
+      },
+      importance: 0.85,
+      contextTags: {
+        weekKey,
+        forumSlug: top.topic.category?.slug || null,
+        source: "adversarial_learning",
+        learningId: record.id,
+      },
+    },
+  });
+
+  return { skipped: false, record, candidate: top.topic, relevance: top.relevance };
+}
+
+// ======== 加载所有 bot 配置 + 角色 + persona ========
+async function loadAllExpertBots() {
+  const prisma = getPrisma();
+  const botUsers = await prisma.user.findMany({
+    where: { isBot: true },
+    include: { botConfig: true },
+  });
+  // 过滤出有 config 的"非路人"专家 bot
+  return botUsers
+    .filter((u) => u.botConfig && (!u.botConfig.personality || !u.botConfig.personality?.role || u.botConfig.personality?.role !== "passerby"))
+    .map((u) => ({ user: u, config: u.botConfig }));
+}
+
+async function loadPersonaForBot(botConfigId) {
+  const prisma = getPrisma();
+  return prisma.botPersona.findUnique({ where: { botId: botConfigId } });
+}
+
+// 尝试从 bot-characters.json 加载角色定义（仅用于 LLM 提示）
+async function loadBotCharMap() {
+  try {
+    const fs = await import("fs");
+    const path = await import("path");
+    const url = await import("url");
+    const { readFileSync } = fs;
+    const { resolve, dirname } = path;
+    const { fileURLToPath } = url;
+    const __dirname = dirname(fileURLToPath(import.meta.url));
+    const p = resolve(__dirname, "..", "..", "data", "bot-characters.json");
+    const raw = readFileSync(p, "utf-8");
+    const { characters } = JSON.parse(raw);
+    const map = {};
+    for (const c of characters) map[c.key] = c;
+    return map;
+  } catch {
+    return {};
+  }
+}
+
+// ======== 周调度入口 ========
+/**
+ * 给所有专家 bot 跑一次对抗学习
+ * @param options
+ *   - weekKey: 强制指定周 key（默认当前 ISO 周）
+ *   - lookbackDays: 候选回看窗口（默认 7）
+ *   - botConfigIds: 限定 bot id 列表
+ *   - skipExisting: 已有本周学习时跳过（默认 true）
+ */
+export async function runWeeklyAdversarialLearning(options = {}) {
+  const prisma = getPrisma();
+  const weekKey = options.weekKey || getISOWeekKey();
+  const botCharMap = await loadBotCharMap();
+
+  const bots = await loadAllExpertBots();
+  const targetBots = options.botConfigIds
+    ? bots.filter((b) => options.botConfigIds.includes(b.config.id))
+    : bots;
+
+  const results = [];
+  for (const { user, config } of targetBots) {
+    try {
+      const persona = await loadPersonaForBot(config.id);
+      const emailPrefix = (user.email || "").split("@")[0] || "";
+      const key = emailPrefix.replace(/^bot_/, "");
+      user._botChar = botCharMap[key] || { displayName: user.name, personality: {} };
+
+      const r = await learnForBot(user, config, persona, {
+        weekKey,
+        lookbackDays: options.lookbackDays ?? LEARNING_CONFIG.lookbackDays,
+      });
+      results.push({ botId: config.id, botName: user.name, ...r });
+    } catch (err) {
+      results.push({ botId: config.id, botName: user.name, error: err.message });
+    }
+  }
+
+  // 把过期的 learning 标记 expired
+  await prisma.botAdversarialLearning.updateMany({
+    where: {
+      status: "active",
+      expiresAt: { lt: new Date() },
+    },
+    data: { status: "expired" },
+  });
+
+  const summary = {
+    weekKey,
+    botCount: targetBots.length,
+    learned: results.filter((r) => !r.skipped && !r.error).length,
+    skipped: results.filter((r) => r.skipped).length,
+    errors: results.filter((r) => r.error).length,
+    results,
+  };
+  return summary;
+}
+
+// ======== 在 prompt 中使用 ========
+/**
+ * 拉取某 bot 当前生效的对抗学习（status=active 且未过期），取最近一条
+ */
+export async function getActiveLearningForBot(botConfigId) {
+  const prisma = getPrisma();
+  return prisma.botAdversarialLearning.findFirst({
+    where: {
+      botId: botConfigId,
+      status: "active",
+      OR: [{ expiresAt: null }, { expiresAt: { gt: new Date() } }],
+    },
+    orderBy: { learnedAt: "desc" },
+  });
+}
+
+/**
+ * 把 learning 渲染成可注入 prompt 的 block
+ */
+export function buildAdversarialBlock(learning) {
+  if (!learning) return "";
+  const lines = [];
+  lines.push(`[对抗学习参考 · ${learning.weekKey}]`);
+  if (learning.sourceTitle) {
+    lines.push(`- 高互动真人帖：${learning.sourceTitle}`);
+  }
+  if (learning.sourceUserName) {
+    lines.push(`- 真人作者：${learning.sourceUserName}`);
+  }
+  if (learning.forumSlug) {
+    lines.push(`- 板块：${learning.forumSlug}`);
+  }
+  const m = learning.sourceMetrics || {};
+  if (m.replyCount !== undefined) {
+    lines.push(
+      `- 互动数据：${m.replyCount || 0} 回复 / ${m.humanReplies || 0} 真人回复 / ${m.likeCount || 0} 赞 / ${m.viewCount || 0} 浏览`
+    );
+  }
+  if (Array.isArray(learning.learnCategories) && learning.learnCategories.length > 0) {
+    lines.push(`- 可借鉴角度：${learning.learnCategories.join("、")}`);
+  }
+  if (learning.learnedInsight) {
+    lines.push(`- 洞察：${learning.learnedInsight}`);
+  }
+  lines.push("- 提示：本周发帖/回复时可参考上述真人帖的切入角度、表达方式，但不要直接抄袭内容。");
+  return `\n${lines.join("\n")}\n`;
+}
+
+/**
+ * 标记 learning 已被使用一次（用于统计触达率）
+ */
+export async function markLearningUsed(learningId) {
+  if (!learningId) return;
+  const prisma = getPrisma();
+  await prisma.botAdversarialLearning.update({
+    where: { id: learningId },
+    data: { usedCount: { increment: 1 } },
+  });
+}
+
+/**
+ * 优雅关闭 prisma
+ */
+export async function disconnectAdversarialLearning() {
+  if (_prisma) {
+    await _prisma.$disconnect();
+    _prisma = null;
+  }
+}