feat: 首次推送到Gitea - 完整项目代码 + 安全加固 + 知识库

scripts/lib/bot-persona.mjs

@@ -8,6 +8,7 @@
 import { PrismaClient } from "@prisma/client";
 import { PrismaMariaDb } from "@prisma/adapter-mariadb";
 import "dotenv/config";
+import { tokenize } from "./chinese-text.mjs";
 
 const base = (process.env.DATABASE_URL || "").replace("mysql://", "mariadb://");
 const sep = base.includes("?") ? "&" : "?";
@@ -29,21 +30,6 @@ const TOP_HUMAN_USERS = 5;
 const TOP_TOPIC_TYPES = 5;
 const LAST_TITLES_COUNT = 5;
 
-const STOP_WORDS = new Set([
-  "的", "了", "是", "在", "和", "与", "或", "也", "都", "就", "不", "没", "要", "我", "你", "他", "她", "它", "们",
-  "这", "那", "有", "为", "到", "对", "及", "等", "把", "被", "从", "向", "以", "其", "之", "于", "上", "下",
-  "里", "外", "中", "一个", "一些", "我们", "你们", "他们", "什么", "怎么", "为什么", "啊", "吗", "呢", "吧",
-  "嗯", "哦", "哈", "啦", "嘿", "唉", "这个", "那个", "一个", "真的", "应该", "可能", "觉得", "认为",
-]);
-
-function tokenize(text) {
-  if (!text) return [];
-  return text
-    .replace(/[，。！？、；：""''【】《》（）()\[\]·—\.\,\!\?\;\:\'\"\\\/]/g, " ")
-    .split(/\s+/)
-    .filter((w) => w.length >= 2 && !STOP_WORDS.has(w));
-}
-
 function pickTopWithCount(arr, topN) {
   const counts = new Map();
   for (const item of arr) {