v0.11.1: 权限体系重构 + AI管线优化 + 内容整理

2026-05-22 18:36:41 +08:00
parent 7955d11c52
commit b33fe1e8e1
397 changed files with 30940 additions and 161 deletions
@@ -0,0 +1,373 @@
+import { PrismaClient } from "@prisma/client";
+import { PrismaMariaDb } from "@prisma/adapter-mariadb";
+import { Octokit } from "octokit";
+import OpenAI from "openai";
+import "dotenv/config";
+
+const base = process.env.DATABASE_URL || "mysql://localhost:3306/zhuiguang_ai?charset=utf8mb4";
+const sep = base.includes("?") ? "&" : "?";
+const connectionString = `${base}${sep}connection_limit=20&pool_timeout=30`;
+const adapter = new PrismaMariaDb(connectionString);
+const prisma = new PrismaClient({ adapter });
+
+const octokit = new Octokit({
+  auth: process.env.GITHUB_TOKEN || undefined,
+});
+
+const openai = new OpenAI({
+  apiKey: process.env.DEEPSEEK_API_KEY,
+  baseURL: "https://api.deepseek.com/v1",
+});
+
+const CATEGORY_KEYWORDS = {
+  "AI Agent": ["agent", "autonomous", "workflow", "tool call"],
+  "代码生成": ["code", "programming", "developer", "copilot"],
+  "测试": ["test", "testing", "qa", "quality"],
+  "文档": ["documentation", "docs", "readme"],
+  "安全": ["security", "vulnerability", "safety", "alignment"],
+  "DevOps与部署": ["deployment", "deploy", "devops", "ci/cd", "docker", "kubernetes"],
+  "RAG": ["rag", "retrieval", "vector", "embedding"],
+  "AI/ML工程": ["training", "fine-tun", "model", "machine learning", "ml"],
+  "多模态": ["multimodal", "vision", "image", "video", "audio"],
+  "Prompt工程": ["prompt", "prompting", "instruction"],
+};
+
+const NON_AI_KEYWORDS = [
+  "book", "guide", "tutorial", "awesome", "interview",
+  "notes", "handbook", "cheatsheet", "cheat-sheet", "roadmap",
+  "everything you need to know", "curated list", "collection",
+];
+
+const AI_TOPICS = [
+  "ai", "artificial-intelligence", "machine-learning", "deep-learning",
+  "llm", "large-language-model", "nlp", "natural-language-processing",
+  "agent", "rag", "gpt", "transformer", "generative-ai", "chatgpt",
+  "langchain", "prompt-engineering", "multimodal", "text-generation",
+  "vector-database", "embedding", "model-training", "fine-tuning",
+];
+
+function isValidAIProject(repo) {
+  const name = (repo.full_name || "").toLowerCase();
+  const desc = (repo.description || "").toLowerCase();
+  const text = `${name} ${desc}`;
+
+  for (const kw of NON_AI_KEYWORDS) {
+    if (text.includes(kw)) {
+      console.log(
+        `   🚫 过滤(非AI): ${repo.full_name} (命中关键词: "${kw}")`
+      );
+      return false;
+    }
+  }
+
+  const language = (repo.language || "").toLowerCase();
+  if (["markdown", "html", "tex", "latex"].includes(language) && repo.stargazers_count < 5000) {
+    console.log(
+      `   🚫 过滤(文档): ${repo.full_name} (语言: ${repo.language}, ⭐${repo.stargazers_count})`
+    );
+    return false;
+  }
+
+  const topics = (repo.topics || []).map((t) => t.toLowerCase());
+  if (topics.length > 0) {
+    const hasAITopic = topics.some((t) =>
+      AI_TOPICS.some((ai) => t.includes(ai))
+    );
+    if (!hasAITopic) {
+      console.log(
+        `   🚫 过滤(无AI主题): ${repo.full_name} (topics: ${topics.join(", ").substring(0, 100)})`
+      );
+      return false;
+    }
+  }
+
+  return true;
+}
+
+function extractJSON(str) {
+  const trimmed = str.trim();
+  const codeBlockMatch = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/);
+  let jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : trimmed;
+  const braceMatch = jsonStr.match(/\{[\s\S]*\}/);
+  if (braceMatch) {
+    try { return JSON.parse(braceMatch[0]); } catch {}
+  }
+  const arrayMatch = jsonStr.match(/\[[\s\S]*\]/);
+  if (arrayMatch) {
+    try { return JSON.parse(arrayMatch[0]); } catch {}
+  }
+  return JSON.parse(jsonStr);
+}
+
+async function guessCategory(name, desc, tags) {
+  const categories = await prisma.skillCategory.findMany({
+    select: { id: true, name: true },
+    orderBy: { sortOrder: "asc" },
+  });
+  if (categories.length === 0) {
+    throw new Error("数据库中没有技能分类，请先创建分类");
+  }
+  const text = `${name} ${desc} ${tags.join(" ")}`.toLowerCase();
+  for (const [categoryName, keywords] of Object.entries(CATEGORY_KEYWORDS)) {
+    for (const kw of keywords) {
+      if (text.includes(kw)) {
+        const match = categories.find((c) => c.name === categoryName);
+        if (match) return match.id;
+      }
+    }
+  }
+  return categories[0].id;
+}
+
+const REVIEW_PROMPT = `你是一个GitHub开源项目评测专家。请基于以下项目信息进行五维深度评测，并以JSON格式返回。务必只返回JSON，不要包含任何其他文字或Markdown格式。
+
+项目信息：
+- 仓库全名：{repoName}
+- 简介：{description}
+- GitHub Stars：{stars}
+- 开源许可证：{license}
+- README摘要（前1500字）：{readmeSummary}
+
+请严格按照以下JSON结构输出：
+{
+  "capability": { "score": 4.5, "summary": "一句话亮点(15字内)", "detail": "80-120字场景能力分析" },
+  "devExp": { "score": 3.5, "summary": "上手体验总结", "detail": "80-120字，包含安装、文档、API评价" },
+  "costLicense": { "score": 4.0, "summary": "成本与许可总结", "detail": "80-120字，包含硬件成本、API定价、许可证风险" },
+  "community": { "score": 4.5, "summary": "社区健康度总结", "detail": "80-120字，包含Issue响应、PR频率、生态描述" },
+  "performance": { "score": 3.5, "summary": "性能稳定性总结", "detail": "80-120字，包含运行效率、稳定性、输出质量" },
+  "overall": 4.0,
+  "tags": ["标签1", "标签2", "标签3"]
+}
+
+评分要求：
+- 分数为1-5之间，可使用0.5步长
+- 综合分应为五个维度分数的算术平均值（保留1位小数）
+- 每个维度的summary需严格控制在15字以内
+- 每个维度的detail需严格控制在80-120字
+- 标签从以下类别中选择最匹配的3-5个：LLM框架, Agent框架, RAG, 提示工程, 多模态, 代码助手, 模型训练, 数据工程, 部署工具, 评测工具, 安全对齐, 其他`;
+
+async function searchHotRepos(query, minStars = 500, maxResults = 10) {
+  const q = `${query} stars:>${minStars} pushed:>2024-01-01`;
+  const { data } = await octokit.rest.search.repos({
+    q,
+    sort: "stars",
+    order: "desc",
+    per_page: maxResults,
+  });
+  return data.items;
+}
+
+async function getRepoReadme(owner, repo) {
+  try {
+    const { data } = await octokit.rest.repos.getReadme({ owner, repo });
+    return Buffer.from(data.content, "base64").toString("utf-8");
+  } catch {
+    return "";
+  }
+}
+
+async function generateReview(repoName, description, stars, license, readme) {
+  const prompt = REVIEW_PROMPT
+    .replace("{repoName}", repoName)
+    .replace("{description}", description || "暂无描述")
+    .replace("{stars}", stars.toString())
+    .replace("{license}", license || "未知")
+    .replace("{readmeSummary}", readme.substring(0, 1500));
+
+  const completion = await openai.chat.completions.create({
+    model: "deepseek-v4-pro",
+    messages: [{ role: "user", content: prompt }],
+    temperature: 0.3,
+  });
+
+  const content = completion.choices[0]?.message?.content;
+  if (!content) {
+    throw new Error("DeepSeek API 返回内容为空");
+  }
+
+  let review = extractJSON(content);
+  if (review.data && typeof review.data === "object") review = review.data;
+  if (review.result && typeof review.result === "object") review = review.result;
+  if (!review.capability || !review.devExp) {
+    throw new Error("评测JSON结构不完整: " + JSON.stringify(Object.keys(review)).substring(0, 200));
+  }
+
+  const dims = [
+    review.capability.score,
+    review.devExp.score,
+    review.costLicense.score,
+    review.community.score,
+    review.performance.score,
+  ];
+  const avg = dims.reduce((a, b) => a + b, 0) / 5;
+  if (Math.abs(review.overall - avg) > 0.3) {
+    console.warn(
+      `⚠️ 综合分 ${review.overall} 与均分 ${avg.toFixed(1)} 偏差较大，已自动修正为均分`
+    );
+    review.overall = Math.round(avg * 10) / 10;
+  }
+
+  return review;
+}
+
+function generateSlug(fullName) {
+  return fullName
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-|-$/g, "")
+    .substring(0, 100);
+}
+
+async function main() {
+  console.log("🚀 [Auto Review] 开始自动化评测任务...\n");
+
+  const SEARCH_QUERIES = [
+    "ai agent",
+    "llm framework",
+    "rag",
+    "prompt engineering",
+  ];
+  const MIN_STARS = 500;
+  const MAX_PER_QUERY = 5;
+  const DAILY_LIMIT = 8;
+
+  let reviewedCount = 0;
+
+  for (const query of SEARCH_QUERIES) {
+    if (reviewedCount >= DAILY_LIMIT) break;
+
+    console.log(`🔍 搜索关键词: "${query}" (min ${MIN_STARS} stars)`);
+
+    const repos = await searchHotRepos(query, MIN_STARS, MAX_PER_QUERY);
+    console.log(`   找到 ${repos.length} 个仓库\n`);
+
+    for (const repo of repos) {
+      if (reviewedCount >= DAILY_LIMIT) break;
+
+      const existing = await prisma.skill.findFirst({
+        where: { sourceUrl: repo.html_url },
+      });
+
+      if (existing) {
+        console.log(
+          `   ⏭️ 跳过已存在: ${repo.full_name} (⭐${repo.stargazers_count})`
+        );
+        continue;
+      }
+
+      if (!isValidAIProject(repo)) {
+        continue;
+      }
+
+      console.log(
+        `\n📦 [${reviewedCount + 1}/${DAILY_LIMIT}] 处理: ${repo.full_name} (⭐${repo.stargazers_count})`
+      );
+
+      try {
+        console.log("   📖 获取 README...");
+        const [owner, name] = repo.full_name.split("/");
+        const readme = await getRepoReadme(owner, name);
+        console.log(`   ✅ README 长度: ${readme.length} 字符`);
+
+        console.log("   🤖 生成五维评测...");
+        const reviewData = await generateReview(
+          repo.full_name,
+          repo.description || "",
+          repo.stargazers_count,
+          repo.license?.spdx_id || "Unknown",
+          readme
+        );
+        console.log(`   ✅ 综合评分: ${reviewData.overall}`);
+
+        const slug = generateSlug(repo.full_name);
+
+        const slugExists = await prisma.skill.findUnique({ where: { slug } });
+        if (slugExists) {
+          console.log(`   ⏭️ 跳过slug冲突: ${slug}`);
+          continue;
+        }
+
+        const categoryId = await guessCategory(repo.full_name, repo.description || "", reviewData?.tags || []);
+
+        const skill = await prisma.skill.create({
+          data: {
+            name: repo.full_name.substring(0, 100),
+            slug,
+            categoryId,
+            description: repo.description || "",
+            sourceUrl: repo.html_url,
+            sourceType: "github",
+            rating: reviewData.overall,
+            features: {
+              forks: repo.forks_count,
+              language: repo.language,
+              license: repo.license?.spdx_id || null,
+              topics: repo.topics || [],
+            },
+            tags: reviewData.tags || [],
+            status: "draft",
+            lastReviewedAt: new Date(),
+            githubStarsHistory: [
+              {
+                date: new Date().toISOString().split("T")[0],
+                stars: repo.stargazers_count,
+              },
+            ],
+          },
+        });
+        console.log(`   ✅ Skill 已创建 (ID: ${skill.id})`);
+
+        await prisma.skillReview.create({
+          data: {
+            skillId: skill.id,
+            capability: reviewData.capability.score,
+            devExp: reviewData.devExp.score,
+            costLicense: reviewData.costLicense.score,
+            community: reviewData.community.score,
+            performance: reviewData.performance.score,
+            overall: reviewData.overall,
+            reviews: reviewData,
+            status: "DRAFT",
+          },
+        });
+        console.log("   ✅ 评测记录已保存");
+
+        await prisma.reviewGenerationLog.create({
+          data: {
+            skillId: skill.id,
+            status: "SUCCESS",
+            prompt: repo.full_name,
+            response: JSON.stringify(reviewData),
+          },
+        });
+
+        reviewedCount++;
+        console.log("   🎉 完成!\n");
+      } catch (error) {
+        console.error(`   ❌ 失败: ${error.message}`);
+
+        await prisma.reviewGenerationLog.create({
+          data: {
+            skillId: 0,
+            status: "FAILED",
+            prompt: repo.full_name,
+            error: error.message?.substring(0, 500) || "Unknown error",
+          },
+        });
+      }
+    }
+  }
+
+  console.log(
+    `\n🏁 [Auto Review] 完成。今日评测: ${reviewedCount} 个项目`
+  );
+}
+
+main()
+  .catch((e) => {
+    console.error("💥 脚本异常:", e);
+    process.exit(1);
+  })
+  .finally(async () => {
+    await prisma.$disconnect();
+  });