全项目扫描修复: Docker数据卷修复+安全requireAdmin+12页SEO+API白名单+脚本超时+常量提取+假数据删除

scripts/bot-skill-crystallize.mjs

@@ -0,0 +1,411 @@
+import { PrismaClient } from "@prisma/client";
+import { PrismaMariaDb } from "@prisma/adapter-mariadb";
+import { readFileSync } from "fs";
+import { resolve, dirname } from "path";
+import { fileURLToPath } from "url";
+import OpenAI from "openai";
+import "dotenv/config";
+import { withRetry } from "./lib/retry.mjs";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const base = process.env.DATABASE_URL.replace("mysql://", "mariadb://");
+const sep = base.includes("?") ? "&" : "?";
+const connectionString = `${base}${sep}connection_limit=5&pool_timeout=30`;
+const adapter = new PrismaMariaDb(connectionString);
+const prisma = new PrismaClient({ adapter });
+
+const openai = new OpenAI({
+  apiKey: process.env.DEEPSEEK_API_KEY,
+  baseURL: "https://api.deepseek.com/v1",
+});
+const MODEL = process.env.DEEPSEEK_MODEL || "deepseek-chat";
+
+const BOT_DATA_PATH = resolve(__dirname, "..", "data", "bot-characters.json");
+
+const CRYSTALLIZE_TRIGGERS = {
+  minReplies: 5,
+  minHumanReplies: 1,
+  minLikes: 3,
+  lookbackDays: 14,
+  maxTopicsPerBot: 8,
+  minTopicsForCrystallize: 2,
+};
+
+function ts() {
+  return `[${new Date().toISOString()}]`;
+}
+
+function pick(arr) {
+  return arr[Math.floor(Math.random() * arr.length)];
+}
+
+async function loadBotCharacters() {
+  const raw = readFileSync(BOT_DATA_PATH, "utf-8");
+  const { characters } = JSON.parse(raw);
+  const map = {};
+  for (const c of characters) {
+    map[c.key] = c;
+  }
+  return { characters, map };
+}
+
+function buildCrystallizePrompt(botChar, topicSummaries) {
+  const p = botChar.personality;
+  return `你是一个"成功帖子解构师"。你负责把一个数字人最近发的高互动帖子提炼成可复用的发帖技能。
+
+[数字人角色]
+- 名字：【${botChar.displayName}】
+- 身份：${p.identity}
+- 立场：${p.stance}
+- 风格：${p.speakingStyle}
+- 口头禅：${p.catchphrase}
+
+[高互动帖子列表]
+${topicSummaries
+  .map(
+    (t, i) =>
+      `[${i + 1}] 标题：${t.title}\n板块：${t.forumName}\n收到回复：${t.replyCount}（真人${t.humanCount}）\n开头：${t.opening}\n要点：${t.bulletPoints}`
+  )
+    .join("\n\n")}
+
+[任务]
+请从以上${topicSummaries.length}个高互动帖子中提取共性，输出一个可复用的发帖技能。
+
+要求：
+1. 技能名 6-12字，简明扼要
+2. 钩子模板（开头如何吸引人）100-200字，可以含{{topic}}占位符
+3. 正文骨架（段落结构+要点清单）150-300字
+4. 适用场景 30-80字，描述什么类型的话题/事件适合套用
+5. 必须保持【${botChar.displayName}】的说话风格和立场，不能泛化
+6. 如果发现明显"反模式"（如"使用了禁止的句式"），在适用场景里反向说明
+
+[输出格式]
+只输出严格JSON，不要其他任何内容：
+{
+  "name": "技能名",
+  "hookPattern": "钩子模板",
+  "bodyTemplate": "正文骨架",
+  "applicability": "适用场景"
+}`;
+}
+
+async function callDeepSeek(prompt) {
+  const response = await withRetry(() =>
+    openai.chat.completions.create({
+      model: MODEL,
+      messages: [{ role: "user", content: prompt }],
+      temperature: 0.7,
+      max_tokens: 1500,
+    })
+  );
+  const text = response.choices[0]?.message?.content?.trim() || "";
+  const jsonMatch = text.match(/\{[\s\S]*\}/);
+  if (!jsonMatch) throw new Error("无法解析JSON: " + text.slice(0, 200));
+  return JSON.parse(jsonMatch[0]);
+}
+
+async function getCrystallizedTopicIds(botConfigId) {
+  const skills = await prisma.botSkill.findMany({
+    where: { botId: botConfigId },
+    select: { sourceTopicIds: true },
+  });
+  const ids = new Set();
+  for (const s of skills) {
+    const arr = s.sourceTopicIds;
+    if (Array.isArray(arr)) for (const id of arr) ids.add(id);
+  }
+  return ids;
+}
+
+async function findQualifiedTopics(botUser, botConfigId) {
+  const since = new Date(Date.now() - CRYSTALLIZE_TRIGGERS.lookbackDays * 24 * 60 * 60 * 1000);
+  const topics = await prisma.forumTopic.findMany({
+    where: {
+      userId: botUser.id,
+      createdAt: { gt: since },
+    },
+    include: {
+      category: { select: { name: true, slug: true } },
+      posts: {
+        include: { user: { select: { isBot: true } } },
+      },
+    },
+    orderBy: { replyCount: "desc" },
+    take: 30,
+  });
+
+  // 拉 topic 下所有回复的真实 likeCount，与 topic.likeCount 合并作为真实点赞
+  const postIds = topics.flatMap((t) => t.posts.map((p) => p.id));
+  const postLikeAgg = postIds.length > 0
+    ? await prisma.forumPost.groupBy({
+        by: ["id"],
+        where: { id: { in: postIds } },
+        _sum: { likeCount: true },
+      })
+    : [];
+  const postLikeMap = new Map(postLikeAgg.map((r) => [r.id, r._sum.likeCount || 0]));
+
+  return topics
+    .map((t) => {
+      const humanReplies = t.posts.filter((p) => !p.user.isBot).length;
+      // 真实点赞：topic 自身的 likeCount + 所有回复的 likeCount 之和
+      const postLikes = t.posts.reduce((s, p) => s + (postLikeMap.get(p.id) ?? 0), 0);
+      const realLikeCount = (t.likeCount || 0) + postLikes;
+      return {
+        id: t.id,
+        title: t.title,
+        forumName: t.category.name,
+        forumSlug: t.category.slug,
+        content: t.content,
+        replyCount: t._count?.posts ?? t.posts.length,
+        humanCount: humanReplies,
+        likeCount: realLikeCount,
+        opening: t.content.slice(0, 200),
+        bulletPoints: t.content
+          .split(/[。\n]/)
+          .map((s) => s.trim())
+          .filter((s) => s.length > 5 && s.length < 60)
+          .slice(0, 5)
+          .join(" / "),
+      };
+    })
+    .filter(
+      (t) =>
+        t.replyCount >= CRYSTALLIZE_TRIGGERS.minReplies ||
+        t.humanCount >= CRYSTALLIZE_TRIGGERS.minHumanReplies ||
+        t.likeCount >= CRYSTALLIZE_TRIGGERS.minLikes
+    );
+}
+
+async function processBotCrystallize(botChar) {
+  const botUser = await prisma.user.findFirst({
+    where: { email: `bot_${botChar.key}@zhuiguang.ai` },
+  });
+  if (!botUser) return;
+
+  const botConfig = await prisma.botConfig.findUnique({
+    where: { userId: botUser.id },
+  });
+  if (!botConfig) return;
+
+  const alreadyCrystallized = await getCrystallizedTopicIds(botConfig.id);
+
+  const allQualified = await findQualifiedTopics(botUser, botConfig.id);
+  const newQualified = allQualified.filter((t) => !alreadyCrystallized.has(t.id));
+
+  if (newQualified.length < CRYSTALLIZE_TRIGGERS.minTopicsForCrystallize) {
+    if (newQualified.length === 0) return;
+    console.log(
+      `${ts()} ${botChar.displayName}: ${newQualified.length} 个新合格话题 (< ${CRYSTALLIZE_TRIGGERS.minTopicsForCrystallize})，跳过`
+    );
+    return;
+  }
+
+  const topicsForCrystallize = newQualified.slice(0, CRYSTALLIZE_TRIGGERS.maxTopicsPerBot);
+  console.log(
+    `${ts()} ${botChar.displayName}: ${topicsForCrystallize.length} 个高互动话题待解构`
+  );
+
+  const forumGroups = {};
+  for (const t of topicsForCrystallize) {
+    if (!forumGroups[t.forumSlug]) forumGroups[t.forumSlug] = [];
+    forumGroups[t.forumSlug].push(t);
+  }
+
+  for (const [forumSlug, topics] of Object.entries(forumGroups)) {
+    if (topics.length < CRYSTALLIZE_TRIGGERS.minTopicsForCrystallize) continue;
+
+    try {
+      const prompt = buildCrystallizePrompt(botChar, topics);
+      const result = await callDeepSeek(prompt);
+
+      if (!result.name || !result.hookPattern || !result.bodyTemplate) {
+        console.warn(`${ts()}   ${forumSlug} 解构结果字段不全，跳过`);
+        continue;
+      }
+
+      const newSkill = await prisma.botSkill.create({
+        data: {
+          botId: botConfig.id,
+          name: result.name.slice(0, 50),
+          category: forumSlug,
+          hookPattern: result.hookPattern,
+          bodyTemplate: result.bodyTemplate,
+          applicability: result.applicability || null,
+          sourceTopicIds: topics.map((t) => t.id),
+          successRate: 0.5,
+          usageCount: 0,
+          avgReplies: topics.reduce((s, t) => s + t.replyCount, 0) / topics.length,
+          avgLikes: topics.reduce((s, t) => s + t.likeCount, 0) / topics.length,
+          version: 1,
+          isActive: true,
+        },
+      });
+
+      await prisma.botMemory.create({
+        data: {
+          botId: botConfig.id,
+          memoryType: "skill_usage",
+          layer: "LONGTERM",
+          content: {
+            action: "crystallize_skill",
+            skillId: newSkill.id,
+            skillName: newSkill.name,
+            category: forumSlug,
+            sourceTopicCount: topics.length,
+            sourceTopicIds: topics.map((t) => t.id),
+          },
+          importance: 0.9,
+          contextTags: {
+            skillId: newSkill.id,
+            forumSlug,
+            source: "skill_crystallize",
+          },
+        },
+      });
+
+      await prisma.taskLog.create({
+        data: {
+          taskKey: "bot-skill-crystallize",
+          taskName: "Bot技能结晶",
+          status: "success",
+          startedAt: new Date(),
+          finishedAt: new Date(),
+          triggerBy: "cron",
+          result: {
+            botKey: botChar.key,
+            forumSlug,
+            skillId: newSkill.id,
+            skillName: newSkill.name,
+            sourceCount: topics.length,
+          },
+        },
+      });
+
+      console.log(
+        `${ts()}   ✅ 结晶 [${forumSlug}] "${newSkill.name}" (来源 ${topics.length} 帖)`
+      );
+    } catch (err) {
+      console.error(`${ts()}   ${forumSlug} 结晶失败:`, err.message);
+      await prisma.taskLog.create({
+        data: {
+          taskKey: "bot-skill-crystallize",
+          taskName: "Bot技能结晶",
+          status: "error",
+          startedAt: new Date(),
+          finishedAt: new Date(),
+          triggerBy: "cron",
+          error: err.message,
+          result: { botKey: botChar.key, forumSlug },
+        },
+      });
+    }
+  }
+}
+
+// 把已经"沉淀"过的技能（基于 sourceTopicIds）回查最近表现：
+// 1) 找到 sourceTopicIds 中每个话题的最新点赞/回复数
+// 2) 与技能里记录的 avgReplies/avgLikes 对比，更新 successRate
+// 3) 对 successRate < 0.2 且 usageCount >= 5 的技能，标记 isActive=false（暂时停用）
+// 返回被回收/调整的技能数
+async function recycleSkillOutcomes() {
+  const skills = await prisma.botSkill.findMany({
+    where: { isActive: true },
+    select: { id: true, sourceTopicIds: true, avgReplies: true, avgLikes: true, usageCount: true, successRate: true },
+  });
+  if (skills.length === 0) return 0;
+
+  let adjusted = 0;
+  for (const skill of skills) {
+    const ids = Array.isArray(skill.sourceTopicIds) ? skill.sourceTopicIds : [];
+    if (ids.length === 0) continue;
+    const topics = await prisma.forumTopic.findMany({
+      where: { id: { in: ids } },
+      select: { replyCount: true, likeCount: true },
+    });
+    if (topics.length === 0) continue;
+    const newAvgReplies = topics.reduce((s, t) => s + t.replyCount, 0) / topics.length;
+    const newAvgLikes = topics.reduce((s, t) => s + t.likeCount, 0) / topics.length;
+    // 简单 successRate = 当前平均点赞 / 5，上限 1
+    const newSuccessRate = Math.min(1, newAvgLikes / 5);
+    const update = {
+      avgReplies: newAvgReplies,
+      avgLikes: newAvgLikes,
+      successRate: newSuccessRate,
+    };
+    if (skill.usageCount >= 5 && newSuccessRate < 0.2) {
+      update.isActive = false;
+    }
+    await prisma.botSkill.update({
+      where: { id: skill.id },
+      data: update,
+    });
+    adjusted++;
+  }
+  return adjusted;
+}
+
+async function main() {
+  console.log(`${ts()} Bot Skill Crystallize starting...`);
+
+  if (!process.env.DEEPSEEK_API_KEY) {
+    throw new Error("DEEPSEEK_API_KEY 未配置");
+  }
+  if (!process.env.DATABASE_URL) {
+    throw new Error("DATABASE_URL 未配置");
+  }
+
+  const { characters } = await loadBotCharacters();
+  const botChars = characters.filter((c) => !c.role || c.role !== "passerby");
+  console.log(`${ts()} Loaded ${botChars.length} expert bots.`);
+
+  let processed = 0;
+  for (const botChar of botChars) {
+    try {
+      await processBotCrystallize(botChar);
+      processed++;
+    } catch (err) {
+      console.error(`${ts()} Error processing ${botChar.key}:`, err.message);
+    }
+  }
+
+  console.log(`${ts()} Recycle step: 回收成熟技能的实际表现...`);
+  const recycled = await recycleSkillOutcomes();
+  console.log(`${ts()} Recycle done: ${recycled} 技能已回收`);
+
+  await prisma.taskLog.create({
+    data: {
+      taskKey: "bot-skill-crystallize",
+      taskName: "Bot技能结晶",
+      status: "success",
+      startedAt: new Date(),
+      finishedAt: new Date(),
+      triggerBy: "cron",
+      result: { processedBots: processed, recycledSkills: recycled },
+    },
+  });
+
+  console.log(`${ts()} Bot Skill Crystallize done. Processed ${processed} bots, recycled ${recycled} skills.`);
+}
+
+main()
+  .catch(async (e) => {
+    console.error(`${ts()} Fatal error:`, e);
+    try {
+      await prisma.taskLog.create({
+        data: {
+          taskKey: "bot-skill-crystallize",
+          taskName: "Bot技能沉淀",
+          status: "failed",
+          startedAt: new Date(),
+          finishedAt: new Date(),
+          triggerBy: "cron",
+          error: (e && e.message ? e.message : String(e)).slice(0, 1000),
+        },
+      });
+    } catch {}
+    process.exit(1);
+  })
+  .finally(async () => {
+    await prisma.$disconnect();
+  });