372 lines
16 KiB
JavaScript
372 lines
16 KiB
JavaScript
import { PrismaClient } from "@prisma/client";
|
||
import { PrismaMariaDb } from "@prisma/adapter-mariadb";
|
||
import { Octokit } from "octokit";
|
||
import OpenAI from "openai";
|
||
import "dotenv/config";
|
||
import { withRetry } from "./lib/retry.mjs";
|
||
|
||
const rawUrl = process.env.DATABASE_URL;
|
||
if (!rawUrl) { console.error("DATABASE_URL not set"); process.exit(1); }
|
||
const base = rawUrl.replace("mysql://", "mariadb://");
|
||
const sep = base.includes("?") ? "&" : "?";
|
||
const connectionString = `${base}${sep}connection_limit=20&pool_timeout=30`;
|
||
const adapter = new PrismaMariaDb(connectionString);
|
||
const prisma = new PrismaClient({ adapter });
|
||
|
||
const octokit = new Octokit({
|
||
auth: process.env.GITHUB_TOKEN || undefined,
|
||
});
|
||
|
||
const openai = new OpenAI({
|
||
apiKey: process.env.DEEPSEEK_API_KEY,
|
||
baseURL: "https://api.deepseek.com",
|
||
timeout: 120000,
|
||
maxRetries: 2,
|
||
});
|
||
|
||
const REVIEW_PROMPT = `你是一个GitHub开源项目评测专家。请基于以下项目信息进行五维深度评测，并以JSON格式返回。务必只返回JSON，不要包含任何其他文字或Markdown格式。
|
||
|
||
项目信息：
|
||
- 仓库全名：{repoName}
|
||
- 简介：{description}
|
||
- GitHub Stars：{stars}
|
||
- 开源许可证：{license}
|
||
- README摘要（前1500字）：{readmeSummary}
|
||
|
||
请严格按照以下JSON结构输出：
|
||
{
|
||
"capability": { "score": 4.5, "summary": "一句话亮点(15字内)", "detail": "80-120字场景能力分析" },
|
||
"devExp": { "score": 3.5, "summary": "上手体验总结", "detail": "80-120字，包含安装、文档、API评价" },
|
||
"costLicense": { "score": 4.0, "summary": "成本与许可总结", "detail": "80-120字，包含硬件成本、API定价、许可证风险" },
|
||
"community": { "score": 4.5, "summary": "社区健康度总结", "detail": "80-120字，包含Issue响应、PR频率、生态描述" },
|
||
"performance": { "score": 3.5, "summary": "性能稳定性总结", "detail": "80-120字，包含运行效率、稳定性、输出质量" },
|
||
"overall": 4.0,
|
||
"tags": ["标签1", "标签2", "标签3"]
|
||
}
|
||
|
||
评分要求：
|
||
- 分数为1-5之间，可使用0.5步长
|
||
- 综合分应为五个维度分数的算术平均值（保留1位小数）
|
||
- 每个维度的summary需严格控制在15字以内
|
||
- 每个维度的detail需严格控制在80-120字
|
||
- 标签从以下类别中选择最匹配的3-5个：LLM框架, Agent框架, RAG, 提示工程, 多模态, 代码助手, 模型训练, 数据工程, 部署工具, 评测工具, 安全对齐, 其他`;
|
||
|
||
// 发现策略：查询池按天轮转 + 多排序/翻页 + 滚动时间窗
|
||
// 旧实现固定 10 条查询 × sort:stars desc × per_page 5 → 每天返回同样 5 个高星仓库，全部"已存在" → 连续多日 0 产出
|
||
const SEARCH_QUERIES = [
|
||
"ai agent framework", "llm framework", "rag framework", "prompt engineering tool",
|
||
"ai code assistant", "multimodal ai", "ai model training", "ai deployment tool",
|
||
"ai evaluation benchmark", "ai safety alignment",
|
||
"llm application", "ai agent tool", "mcp server", "ai workflow automation",
|
||
"text to speech ai", "ai image generation", "ai video generation", "vector database",
|
||
"ai coding agent", "ai data analysis", "ai browser automation", "llm observability",
|
||
"ai document processing", "ai speech recognition",
|
||
];
|
||
|
||
const QUERIES_PER_RUN = 5; // 每次运行取 5 条查询（按天轮转，4 天覆盖全池）
|
||
const PAGE_SIZE = 20; // 单个查询每页 20 条
|
||
const MAX_PAGES = 2; // 热门存量最多翻 2 页
|
||
const PUSHED_MONTHS = 18; // 只看近 18 个月有提交的仓库
|
||
const MIN_STARS = 1000; // 存量热门门槛
|
||
const NEW_MIN_STARS = 300; // 新项目门槛（近 12 个月创建）
|
||
const NEW_CREATED_MONTHS = 12;
|
||
|
||
const CATEGORY_KEYWORDS = {
|
||
"MCP 服务器": ["mcp", "model context protocol"],
|
||
"RAG 系统": ["rag", "retrieval", "vector", "embedding"],
|
||
"Prompt 工程": ["prompt", "prompting", "instruction"],
|
||
"Agent 编排": ["agent", "autonomous", "multi-agent", "tool call", "workflow"],
|
||
"代码生成": ["code", "coding", "programming", "developer", "copilot"],
|
||
"测试": ["test", "testing", "qa", "benchmark", "evaluation"],
|
||
"文档": ["documentation", "docs", "readme"],
|
||
"安全": ["security", "vulnerability", "safety", "alignment"],
|
||
"DevOps 与部署": ["deployment", "deploy", "devops", "ci/cd", "docker", "kubernetes"],
|
||
"模型训练": ["fine-tun", "finetune", "lora", "training", "pretrain"],
|
||
"模型部署": ["inference", "serving", "vllm", "ollama", "quantization"],
|
||
"AI/ML 工程": ["machine learning", "deep learning", "pytorch", "tensorflow", "llm", "model"],
|
||
"架构与设计": ["architecture", "design pattern", "spec"],
|
||
"开发工具": ["cli", "sdk", "toolkit", "browser automation", "playwright"],
|
||
};
|
||
|
||
const NON_AI_KEYWORDS = [
|
||
"book", "guide", "tutorial", "awesome", "interview",
|
||
"notes", "handbook", "cheatsheet", "cheat-sheet", "roadmap",
|
||
"everything you need to know", "curated list", "collection",
|
||
];
|
||
|
||
const AI_TOPICS = [
|
||
"ai", "artificial-intelligence", "machine-learning", "deep-learning",
|
||
"llm", "large-language-model", "nlp", "natural-language-processing",
|
||
"agent", "rag", "gpt", "transformer", "generative-ai", "chatgpt",
|
||
"langchain", "prompt-engineering", "multimodal", "text-generation",
|
||
"vector-database", "embedding", "model-training", "fine-tuning",
|
||
];
|
||
|
||
function isValidAIProject(repo) {
|
||
const name = (repo.full_name || "").toLowerCase();
|
||
const desc = (repo.description || "").toLowerCase();
|
||
const text = `${name} ${desc}`;
|
||
|
||
for (const kw of NON_AI_KEYWORDS) {
|
||
if (text.includes(kw)) {
|
||
console.log(
|
||
` 🚫 过滤(非AI): ${repo.full_name} (命中关键词: "${kw}")`
|
||
);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
const language = (repo.language || "").toLowerCase();
|
||
if (["markdown", "html", "tex", "latex"].includes(language) && repo.stargazers_count < 5000) {
|
||
console.log(
|
||
` 🚫 过滤(文档): ${repo.full_name} (语言: ${repo.language}, ⭐${repo.stargazers_count})`
|
||
);
|
||
return false;
|
||
}
|
||
|
||
const topics = (repo.topics || []).map((t) => t.toLowerCase());
|
||
if (topics.length > 0) {
|
||
const hasAITopic = topics.some((t) =>
|
||
AI_TOPICS.some((ai) => t.includes(ai))
|
||
);
|
||
if (!hasAITopic) {
|
||
console.log(
|
||
` 🚫 过滤(无AI主题): ${repo.full_name} (topics: ${topics.join(", ").substring(0, 100)})`
|
||
);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
function extractJSON(str) {
|
||
const trimmed = str.trim();
|
||
const codeBlockMatch = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/);
|
||
let jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : trimmed;
|
||
const braceMatch = jsonStr.match(/\{[\s\S]*\}/);
|
||
if (braceMatch) {
|
||
try { return JSON.parse(braceMatch[0]); } catch {}
|
||
}
|
||
const arrayMatch = jsonStr.match(/\[[\s\S]*\]/);
|
||
if (arrayMatch) {
|
||
try { return JSON.parse(arrayMatch[0]); } catch {}
|
||
}
|
||
return JSON.parse(jsonStr);
|
||
}
|
||
|
||
async function guessCategory(name, desc, tags) {
|
||
const categories = await prisma.skillCategory.findMany({
|
||
select: { id: true, name: true },
|
||
orderBy: { sortOrder: "asc" },
|
||
});
|
||
if (categories.length === 0) {
|
||
throw new Error("数据库中没有技能分类，请先创建分类");
|
||
}
|
||
const text = `${name} ${desc} ${tags.join(" ")}`.toLowerCase();
|
||
for (const [categoryName, keywords] of Object.entries(CATEGORY_KEYWORDS)) {
|
||
for (const kw of keywords) {
|
||
if (text.includes(kw)) {
|
||
const match = categories.find((c) => c.name === categoryName);
|
||
if (match) return match.id;
|
||
}
|
||
}
|
||
}
|
||
return categories[0].id;
|
||
}
|
||
|
||
function genSlug(name) {
|
||
return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").substring(0, 100);
|
||
}
|
||
|
||
function monthsAgo(n) {
|
||
return new Date(Date.now() - n * 30 * 24 * 3600 * 1000).toISOString().slice(0, 10);
|
||
}
|
||
|
||
async function searchRepos(query, { minStars, sort, page, createdAfter }) {
|
||
const parts = [query, `stars:>${minStars}`, `pushed:>${monthsAgo(PUSHED_MONTHS)}`];
|
||
if (createdAfter) parts.push(`created:>${createdAfter}`);
|
||
const q = parts.join(" ");
|
||
const { data } = await withRetry(async () => {
|
||
return await octokit.rest.search.repos({ q, sort, order: "desc", per_page: PAGE_SIZE, page });
|
||
}, { maxRetries: 3, label: `GitHub Search "${q}"` });
|
||
return data.items || [];
|
||
}
|
||
|
||
// 对每条查询做两轮搜索：① 存量热门（按 updated 排序 + 翻页，结果随时间自然浮动）
|
||
// ② 近 12 个月创建的新兴项目（星标 300+），避免只看到固定的一批老牌高星仓库
|
||
async function collectCandidates(queries) {
|
||
const map = new Map();
|
||
let searchFailed = 0;
|
||
for (const query of queries) {
|
||
const passes = [
|
||
{ minStars: MIN_STARS, sort: "updated", pages: Array.from({ length: MAX_PAGES }, (_, i) => i + 1) },
|
||
{ minStars: NEW_MIN_STARS, sort: "stars", createdAfter: monthsAgo(NEW_CREATED_MONTHS), pages: [1] },
|
||
];
|
||
for (const pass of passes) {
|
||
for (const page of pass.pages) {
|
||
try {
|
||
const items = await searchRepos(query, { minStars: pass.minStars, sort: pass.sort, page, createdAfter: pass.createdAfter });
|
||
for (const repo of items) {
|
||
if (!map.has(repo.full_name)) map.set(repo.full_name, repo);
|
||
}
|
||
} catch (e) {
|
||
console.error(` ❌ 搜索失败 "${query}" (${pass.sort} p${page}): ${e.message}`);
|
||
searchFailed++;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return { candidates: [...map.values()], searchFailed };
|
||
}
|
||
|
||
async function getReadme(owner, repo) {
|
||
try {
|
||
const { data } = await withRetry(async () => {
|
||
return await octokit.rest.repos.getReadme({ owner, repo });
|
||
}, { maxRetries: 3, label: `GitHub README ${owner}/${repo}` });
|
||
return Buffer.from(data.content, "base64").toString("utf-8");
|
||
} catch { return ""; }
|
||
}
|
||
|
||
async function genReview(repoName, desc, stars, license, readme) {
|
||
const prompt = REVIEW_PROMPT
|
||
.replace("{repoName}", repoName).replace("{description}", desc || "暂无描述")
|
||
.replace("{stars}", stars.toString()).replace("{license}", license || "未知")
|
||
.replace("{readmeSummary}", readme.substring(0, 1500));
|
||
const { choices } = await withRetry(async () => {
|
||
return await openai.chat.completions.create({
|
||
model: process.env.DEEPSEEK_MODEL || "deepseek-v4-pro",
|
||
messages: [{ role: "user", content: prompt }],
|
||
temperature: 0.3,
|
||
// DeepSeek V4 的 reasoning token 计入 max_tokens，预算不足会返回空 content（历史踩坑）
|
||
max_tokens: 16384,
|
||
}, { signal: AbortSignal.timeout(120000) });
|
||
}, { maxRetries: 3, label: "DeepSeek Review API" });
|
||
const content = choices[0]?.message?.content;
|
||
if (!content) throw new Error("Empty response");
|
||
let review = extractJSON(content);
|
||
if (review.data && typeof review.data === "object") review = review.data;
|
||
if (review.result && typeof review.result === "object") review = review.result;
|
||
if (!review.capability || !review.devExp || !review.costLicense || !review.community || !review.performance) {
|
||
throw new Error("评测JSON结构不完整: " + JSON.stringify(Object.keys(review)).substring(0, 200));
|
||
}
|
||
const dims = [review.capability.score, review.devExp.score, review.costLicense.score, review.community.score, review.performance.score];
|
||
const avg = dims.reduce((a, b) => a + b, 0) / 5;
|
||
if (Math.abs(review.overall - avg) > 0.3) review.overall = Math.round(avg * 10) / 10;
|
||
return review;
|
||
}
|
||
|
||
async function main() {
|
||
if (!process.env.DEEPSEEK_API_KEY) throw new Error("缺少环境变量: DEEPSEEK_API_KEY");
|
||
if (!process.env.DATABASE_URL) throw new Error("缺少环境变量: DATABASE_URL");
|
||
|
||
const startTime = new Date();
|
||
console.log(`🚀 [Task4] ${new Date().toISOString().split("T")[0]} 发现新AI技能...\n`);
|
||
const DAILY_LIMIT = 30;
|
||
let discovered = 0, skipped = 0, reviewed = 0, failed = 0;
|
||
|
||
const dayIndex = Math.floor(Date.now() / 86400000);
|
||
const startIdx = (dayIndex * QUERIES_PER_RUN) % SEARCH_QUERIES.length;
|
||
const todaysQueries = Array.from({ length: QUERIES_PER_RUN }, (_, i) => SEARCH_QUERIES[(startIdx + i) % SEARCH_QUERIES.length]);
|
||
console.log(`🔍 本轮查询 (${todaysQueries.length}/${SEARCH_QUERIES.length}，按天轮转): ${todaysQueries.join(" / ")}`);
|
||
|
||
const { candidates, searchFailed } = await collectCandidates(todaysQueries);
|
||
failed += searchFailed;
|
||
console.log(` 候选仓库 ${candidates.length} 个（已去重）\n`);
|
||
|
||
for (const repo of candidates) {
|
||
if (discovered >= DAILY_LIMIT) break;
|
||
const ex = await prisma.skill.findFirst({ where: { sourceUrl: repo.html_url } });
|
||
if (ex) { skipped++; continue; }
|
||
const exP = await prisma.pendingSkill.findFirst({ where: { sourceUrl: repo.html_url } });
|
||
if (exP) { skipped++; continue; }
|
||
const slug = genSlug(repo.full_name);
|
||
const slugEx = await prisma.skill.findUnique({ where: { slug } });
|
||
if (slugEx) { skipped++; continue; }
|
||
if (!isValidAIProject(repo)) { skipped++; continue; }
|
||
|
||
console.log(`\n📦 [${discovered + 1}/${DAILY_LIMIT}] ${repo.full_name} (⭐${repo.stargazers_count})`);
|
||
try {
|
||
const [owner, name] = repo.full_name.split("/");
|
||
const readme = await getReadme(owner, name);
|
||
let reviewData = null;
|
||
try { reviewData = await genReview(repo.full_name, repo.description || "", repo.stargazers_count, repo.license?.spdx_id || "Unknown", readme); console.log(` ✅ 预评测: ${reviewData.overall}`); reviewed++; }
|
||
catch (e) { console.log(` ⚠️ 预评测失败: ${e.message}`); }
|
||
|
||
const categoryId = await guessCategory(repo.full_name, repo.description || "", reviewData?.tags || []);
|
||
const pendingSkill = await prisma.pendingSkill.create({
|
||
data: {
|
||
name: repo.full_name.substring(0, 100), slug, categoryId,
|
||
description: repo.description || "", sourceUrl: repo.html_url, sourceType: "github",
|
||
rating: reviewData?.overall || 0,
|
||
features: {
|
||
forks: repo.forks_count,
|
||
language: repo.language,
|
||
license: repo.license?.spdx_id || null,
|
||
topics: repo.topics || [],
|
||
stars: repo.stargazers_count,
|
||
reviewData: reviewData || null,
|
||
},
|
||
tags: reviewData?.tags || [],
|
||
status: "pending",
|
||
},
|
||
});
|
||
console.log(` ✅ PendingSkill #${pendingSkill.id}`);
|
||
|
||
await prisma.reviewGenerationLog.create({
|
||
data: { skillId: 0, status: "SUCCESS", prompt: `task4-discover: ${repo.full_name}`, response: reviewData ? JSON.stringify(reviewData) : null },
|
||
});
|
||
discovered++;
|
||
} catch (e) {
|
||
console.error(` ❌ 失败: ${e.message}`);
|
||
failed++;
|
||
try {
|
||
await prisma.reviewGenerationLog.create({
|
||
data: { skillId: 0, status: "FAILED", prompt: `task4-discover: ${repo.full_name}`, error: e.message?.substring(0, 500) || "Unknown" },
|
||
});
|
||
} catch {}
|
||
}
|
||
}
|
||
|
||
const endTime = new Date();
|
||
const duration = endTime.getTime() - startTime.getTime();
|
||
const result = { discovered, skipped, reviewed, failed, duration: `${(duration / 1000).toFixed(1)}s` };
|
||
|
||
console.log(`\n🏁 [Task4] 完成: 发现${discovered} / 跳过${skipped} / 评测${reviewed} / 失败${failed}`);
|
||
|
||
await prisma.taskLog.create({
|
||
data: {
|
||
taskKey: "task4-discover-skills",
|
||
taskName: "发现AI技能",
|
||
status: "completed",
|
||
startedAt: startTime,
|
||
finishedAt: endTime,
|
||
duration,
|
||
result,
|
||
error: failed > discovered * 2 ? `失败率过高: ${failed}/${discovered}` : null,
|
||
},
|
||
});
|
||
}
|
||
|
||
main().catch(async (e) => {
|
||
console.error("💥 异常:", e);
|
||
if (prisma) {
|
||
try {
|
||
await prisma.taskLog.create({
|
||
data: {
|
||
taskKey: "task4-discover-skills",
|
||
taskName: "发现AI技能",
|
||
status: "failed",
|
||
startedAt: new Date(),
|
||
finishedAt: new Date(),
|
||
duration: 0,
|
||
error: e.message,
|
||
},
|
||
});
|
||
} catch {}
|
||
await prisma.$disconnect();
|
||
}
|
||
process.exit(1);
|
||
}).finally(() => prisma.$disconnect());
|