zhuiguang-ai/scripts/daily-discover.mjs

import { PrismaClient } from "@prisma/client";
import { PrismaMariaDb } from "@prisma/adapter-mariadb";
import { Octokit } from "octokit";
import OpenAI from "openai";
import "dotenv/config";
import { withRetry } from "./lib/retry.mjs";

const base = process.env.DATABASE_URL || "mysql://localhost:3306/zhuiguang_ai?charset=utf8mb4";
const sep = base.includes("?") ? "&" : "?";
const connectionString = `${base}${sep}connection_limit=20&pool_timeout=30`;
const adapter = new PrismaMariaDb(connectionString);
const prisma = new PrismaClient({ adapter });

const octokit = new Octokit({
  auth: process.env.GITHUB_TOKEN || undefined,
});

const openai = new OpenAI({
  apiKey: process.env.DEEPSEEK_API_KEY,
  baseURL: "https://api.deepseek.com/v1",
});

const REVIEW_PROMPT = `你是一个GitHub开源项目评测专家。请基于以下项目信息进行五维深度评测，并以JSON格式返回。务必只返回JSON，不要包含任何其他文字或Markdown格式。

项目信息：
- 仓库全名：{repoName}
- 简介：{description}
- GitHub Stars：{stars}
- 开源许可证：{license}
- README摘要（前1500字）：{readmeSummary}

请严格按照以下JSON结构输出：
{
  "capability": { "score": 4.5, "summary": "一句话亮点(15字内)", "detail": "80-120字场景能力分析" },
  "devExp": { "score": 3.5, "summary": "上手体验总结", "detail": "80-120字，包含安装、文档、API评价" },
  "costLicense": { "score": 4.0, "summary": "成本与许可总结", "detail": "80-120字，包含硬件成本、API定价、许可证风险" },
  "community": { "score": 4.5, "summary": "社区健康度总结", "detail": "80-120字，包含Issue响应、PR频率、生态描述" },
  "performance": { "score": 3.5, "summary": "性能稳定性总结", "detail": "80-120字，包含运行效率、稳定性、输出质量" },
  "overall": 4.0,
  "tags": ["标签1", "标签2", "标签3"]
}

评分要求：
- 分数为1-5之间，可使用0.5步长
- 综合分应为五个维度分数的算术平均值（保留1位小数）
- 每个维度的summary需严格控制在15字以内
- 每个维度的detail需严格控制在80-120字
- 标签从以下类别中选择最匹配的3-5个：LLM框架, Agent框架, RAG, 提示工程, 多模态, 代码助手, 模型训练, 数据工程, 部署工具, 评测工具, 安全对齐, 其他`;

const SEARCH_QUERIES = [
  "ai agent framework",
  "llm framework",
  "rag framework",
  "prompt engineering tool",
  "ai code assistant",
  "multimodal ai",
  "ai model training",
  "ai deployment tool",
  "ai evaluation benchmark",
  "ai safety alignment",
];

const CATEGORY_KEYWORDS = {
  "AI Agent": ["agent", "autonomous", "workflow", "tool call"],
  "代码生成": ["code", "programming", "developer", "copilot"],
  "测试": ["test", "testing", "qa", "quality"],
  "文档": ["documentation", "docs", "readme"],
  "安全": ["security", "vulnerability", "safety", "alignment"],
  "DevOps与部署": ["deployment", "deploy", "devops", "ci/cd", "docker", "kubernetes"],
  "RAG": ["rag", "retrieval", "vector", "embedding"],
  "AI/ML工程": ["training", "fine-tun", "model", "machine learning", "ml"],
  "多模态": ["multimodal", "vision", "image", "video", "audio"],
  "Prompt工程": ["prompt", "prompting", "instruction"],
};

const NON_AI_KEYWORDS = [
  "book", "guide", "tutorial", "awesome", "interview",
  "notes", "handbook", "cheatsheet", "cheat-sheet", "roadmap",
  "everything you need to know", "curated list", "collection",
];

const AI_TOPICS = [
  "ai", "artificial-intelligence", "machine-learning", "deep-learning",
  "llm", "large-language-model", "nlp", "natural-language-processing",
  "agent", "rag", "gpt", "transformer", "generative-ai", "chatgpt",
  "langchain", "prompt-engineering", "multimodal", "text-generation",
  "vector-database", "embedding", "model-training", "fine-tuning",
];

function isValidAIProject(repo) {
  const name = (repo.full_name || "").toLowerCase();
  const desc = (repo.description || "").toLowerCase();
  const text = `${name} ${desc}`;

  for (const kw of NON_AI_KEYWORDS) {
    if (text.includes(kw)) {
      console.log(
        `   🚫 过滤(非AI): ${repo.full_name} (命中关键词: "${kw}")`
      );
      return false;
    }
  }

  const language = (repo.language || "").toLowerCase();
  if (["markdown", "html", "tex", "latex"].includes(language) && repo.stargazers_count < 5000) {
    console.log(
      `   🚫 过滤(文档): ${repo.full_name} (语言: ${repo.language}, ⭐${repo.stargazers_count})`
    );
    return false;
  }

  const topics = (repo.topics || []).map((t) => t.toLowerCase());
  if (topics.length > 0) {
    const hasAITopic = topics.some((t) =>
      AI_TOPICS.some((ai) => t.includes(ai))
    );
    if (!hasAITopic) {
      console.log(
        `   🚫 过滤(无AI主题): ${repo.full_name} (topics: ${topics.join(", ").substring(0, 100)})`
      );
      return false;
    }
  }

  return true;
}

function extractJSON(str) {
  const trimmed = str.trim();
  const codeBlockMatch = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/);
  let jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : trimmed;
  const braceMatch = jsonStr.match(/\{[\s\S]*\}/);
  if (braceMatch) {
    try { return JSON.parse(braceMatch[0]); } catch {}
  }
  const arrayMatch = jsonStr.match(/\[[\s\S]*\]/);
  if (arrayMatch) {
    try { return JSON.parse(arrayMatch[0]); } catch {}
  }
  return JSON.parse(jsonStr);
}

async function guessCategory(name, desc, tags) {
  const categories = await prisma.skillCategory.findMany({
    select: { id: true, name: true },
    orderBy: { sortOrder: "asc" },
  });
  if (categories.length === 0) {
    throw new Error("数据库中没有技能分类，请先创建分类");
  }
  const text = `${name} ${desc} ${tags.join(" ")}`.toLowerCase();
  for (const [categoryName, keywords] of Object.entries(CATEGORY_KEYWORDS)) {
    for (const kw of keywords) {
      if (text.includes(kw)) {
        const match = categories.find((c) => c.name === categoryName);
        if (match) return match.id;
      }
    }
  }
  return categories[0].id;
}

function genSlug(name) {
  return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").substring(0, 100);
}

async function searchRepos(query, minStars, max) {
  const q = `${query} stars:>${minStars} pushed:>2024-01-01`;
  const { data } = await withRetry(async () => {
    return await octokit.rest.search.repos({ q, sort: "stars", order: "desc", per_page: max });
  }, { maxRetries: 3, label: `GitHub Search "${query}"` });
  return data.items;
}

async function getReadme(owner, repo) {
  try {
    const { data } = await withRetry(async () => {
      return await octokit.rest.repos.getReadme({ owner, repo });
    }, { maxRetries: 3, label: `GitHub README ${owner}/${repo}` });
    return Buffer.from(data.content, "base64").toString("utf-8");
  } catch { return ""; }
}

async function genReview(repoName, desc, stars, license, readme) {
  const prompt = REVIEW_PROMPT
    .replace("{repoName}", repoName).replace("{description}", desc || "暂无描述")
    .replace("{stars}", stars.toString()).replace("{license}", license || "未知")
    .replace("{readmeSummary}", readme.substring(0, 1500));
  const { choices } = await withRetry(async () => {
    return await openai.chat.completions.create({
      model: process.env.DEEPSEEK_MODEL || "deepseek-v4-pro", messages: [{ role: "user", content: prompt }], temperature: 0.3,
    }, { signal: AbortSignal.timeout(120000) });
  }, { maxRetries: 3, label: "DeepSeek Review API" });
  const content = choices[0]?.message?.content;
  if (!content) throw new Error("Empty response");
  let review = extractJSON(content);
  if (review.data && typeof review.data === "object") review = review.data;
  if (review.result && typeof review.result === "object") review = review.result;
  if (!review.capability || !review.devExp || !review.costLicense || !review.community || !review.performance) {
    throw new Error("评测JSON结构不完整: " + JSON.stringify(Object.keys(review)).substring(0, 200));
  }
  const dims = [review.capability.score, review.devExp.score, review.costLicense.score, review.community.score, review.performance.score];
  const avg = dims.reduce((a, b) => a + b, 0) / 5;
  if (Math.abs(review.overall - avg) > 0.3) review.overall = Math.round(avg * 10) / 10;
  return review;
}

async function main() {
  if (!process.env.DEEPSEEK_API_KEY) throw new Error("缺少环境变量: DEEPSEEK_API_KEY");
  if (!process.env.DATABASE_URL) throw new Error("缺少环境变量: DATABASE_URL");

  const startTime = new Date();
  console.log(`🚀 [Task4] ${new Date().toISOString().split("T")[0]} 发现新AI技能...\n`);
  const DAILY_LIMIT = 30;
  let discovered = 0, skipped = 0, reviewed = 0, failed = 0;

  for (const query of SEARCH_QUERIES) {
    if (discovered >= DAILY_LIMIT) break;
    console.log(`🔍 "${query}"`);
    let repos;
    try { repos = await searchRepos(query, 1000, 5); } catch (e) { console.error(`   ❌ 搜索失败: ${e.message}`); failed++; continue; }

    for (const repo of repos) {
      if (discovered >= DAILY_LIMIT) break;
      const ex = await prisma.skill.findFirst({ where: { sourceUrl: repo.html_url } });
      if (ex) { console.log(`   ⏭️ ${repo.full_name} 已存在`); skipped++; continue; }
      const exP = await prisma.pendingSkill.findFirst({ where: { sourceUrl: repo.html_url } });
      if (exP) { console.log(`   ⏭️ ${repo.full_name} 待审核中`); skipped++; continue; }
      const slug = genSlug(repo.full_name);
      const slugEx = await prisma.skill.findUnique({ where: { slug } });
      if (slugEx) { console.log(`   ⏭️ ${repo.full_name} slug冲突`); skipped++; continue; }
      if (!isValidAIProject(repo)) { skipped++; continue; }

      console.log(`\n📦 [${discovered + 1}/${DAILY_LIMIT}] ${repo.full_name} (⭐${repo.stargazers_count})`);
      try {
        const [owner, name] = repo.full_name.split("/");
        const readme = await getReadme(owner, name);
        let reviewData = null;
        try { reviewData = await genReview(repo.full_name, repo.description || "", repo.stargazers_count, repo.license?.spdx_id || "Unknown", readme); console.log(`   ✅ 预评测: ${reviewData.overall}`); reviewed++; }
        catch (e) { console.log(`   ⚠️ 预评测失败: ${e.message}`); }

        const categoryId = await guessCategory(repo.full_name, repo.description || "", reviewData?.tags || []);
        const pendingSkill = await prisma.pendingSkill.create({
          data: {
            name: repo.full_name.substring(0, 100), slug, categoryId,
            description: repo.description || "", sourceUrl: repo.html_url, sourceType: "github",
            rating: reviewData?.overall || 0,
            features: {
              forks: repo.forks_count,
              language: repo.language,
              license: repo.license?.spdx_id || null,
              topics: repo.topics || [],
              stars: repo.stargazers_count,
              reviewData: reviewData || null,
            },
            tags: reviewData?.tags || [],
            status: "pending",
          },
        });
        console.log(`   ✅ PendingSkill #${pendingSkill.id}`);

        await prisma.reviewGenerationLog.create({
          data: { skillId: pendingSkill.id, status: "SUCCESS", prompt: `task4-discover: ${repo.full_name}`, response: reviewData ? JSON.stringify(reviewData) : null },
        });
        discovered++;
      } catch (e) {
        console.error(`   ❌ 失败: ${e.message}`);
        failed++;
        try {
          await prisma.reviewGenerationLog.create({
            data: { skillId: 0, status: "FAILED", prompt: `task4-discover: ${repo.full_name}`, error: e.message?.substring(0, 500) || "Unknown" },
          });
        } catch {}
      }
    }
  }

  const endTime = new Date();
  const duration = endTime.getTime() - startTime.getTime();
  const result = { discovered, skipped, reviewed, failed, duration: `${(duration / 1000).toFixed(1)}s` };

  console.log(`\n🏁 [Task4] 完成: 发现${discovered} / 跳过${skipped} / 评测${reviewed} / 失败${failed}`);

  await prisma.taskLog.create({
    data: {
      taskKey: "task4-discover-skills",
      taskName: "发现AI技能",
      status: "completed",
      startedAt: startTime,
      finishedAt: endTime,
      duration,
      result,
      error: failed > discovered * 2 ? `失败率过高: ${failed}/${discovered}` : null,
    },
  });
}

main().catch(async (e) => {
  console.error("💥 异常:", e);
  if (prisma) {
    try {
      await prisma.taskLog.create({
        data: {
          taskKey: "task4-discover-skills",
          taskName: "发现AI技能",
          status: "failed",
          startedAt: new Date(),
          finishedAt: new Date(),
          duration: 0,
          error: e.message,
        },
      });
    } catch {}
    await prisma.$disconnect();
  }
  process.exit(1);
}).finally(() => prisma.$disconnect());