263 lines
12 KiB
JavaScript
263 lines
12 KiB
JavaScript
import { PrismaClient } from "@prisma/client";
|
||
import { PrismaMariaDb } from "@prisma/adapter-mariadb";
|
||
import OpenAI from "openai";
|
||
import "dotenv/config";
|
||
import { withRetry } from "./lib/retry.mjs";
|
||
|
||
const rawUrl = process.env.DATABASE_URL;
|
||
if (!rawUrl) { console.error("DATABASE_URL not set"); process.exit(1); }
|
||
const base = rawUrl.replace("mysql://", "mariadb://");
|
||
const sep = base.includes("?") ? "&" : "?";
|
||
const connectionString = `${base}${sep}connection_limit=20&pool_timeout=30`;
|
||
const adapter = new PrismaMariaDb(connectionString);
|
||
const prisma = new PrismaClient({ adapter });
|
||
const openai = new OpenAI({ apiKey: process.env.DEEPSEEK_API_KEY, baseURL: "https://api.deepseek.com", timeout: 120000, maxRetries: 2 });
|
||
|
||
const EXTRACT_PROMPT = `你是一个AI工具信息提取助手。从以下科技新闻内容中，提取出现的AI工具/产品信息。只提取明确提到的AI软件工具、AI平台、AI产品，不要提取开源项目库（GitHub仓库）、不要提取硬件、芯片。每个工具返回以下信息（如果信息不足则跳过该工具）：
|
||
|
||
新闻内容：
|
||
{content}
|
||
|
||
返回纯JSON数组：
|
||
[{
|
||
"name": "工具名称",
|
||
"description": "一句话简介（20字以内）",
|
||
"websiteUrl": "官网或产品主页URL",
|
||
"category": "分类关键词（只能从以下选一）：文本生成/图像生成/视频生成/语音音频/代码编程/办公效率/营销创作/数据AI/教育学习/设计制作"
|
||
}]`;
|
||
|
||
function extractJSON(str) {
|
||
const trimmed = str.trim();
|
||
const codeBlockMatch = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/);
|
||
let jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : trimmed;
|
||
const arrayMatch = jsonStr.match(/\[[\s\S]*\]/);
|
||
if (arrayMatch) {
|
||
try { return JSON.parse(arrayMatch[0]); } catch {}
|
||
}
|
||
return JSON.parse(jsonStr);
|
||
}
|
||
|
||
async function fetchRSS(url) {
|
||
try {
|
||
const res = await withRetry(async () => {
|
||
return await fetch(url, {
|
||
headers: { "User-Agent": "Mozilla/5.0 (compatible; ToolBot/1.0)" },
|
||
signal: AbortSignal.timeout(10000),
|
||
});
|
||
}, { maxRetries: 3, label: `RSS ${url}` });
|
||
if (!res.ok) return [];
|
||
const xml = await res.text();
|
||
const items = [];
|
||
const regex = /<item>([\s\S]*?)<\/item>/g;
|
||
let m;
|
||
while ((m = regex.exec(xml)) !== null) {
|
||
const b = m[1];
|
||
const t = (b.match(/<title><!\[CDATA\[([\s\S]*?)\]\]><\/title>/) || b.match(/<title>([\s\S]*?)<\/title>/))?.[1] || "";
|
||
const d = (b.match(/<description><!\[CDATA\[([\s\S]*?)\]\]><\/description>/) || b.match(/<description>([\s\S]*?)<\/description>/))?.[1] || "";
|
||
items.push({ title: t.trim(), content: d.replace(/<[^>]+>/g, "").trim().substring(0, 500) });
|
||
}
|
||
return items.slice(0, 20);
|
||
} catch { return []; }
|
||
}
|
||
|
||
async function checkWebsite(url) {
|
||
try {
|
||
const res = await withRetry(async () => {
|
||
return await fetch(url, { method: "GET", signal: AbortSignal.timeout(8000), redirect: "follow" });
|
||
}, { maxRetries: 2, label: `Website ${url}` });
|
||
return res.ok;
|
||
} catch { return false; }
|
||
}
|
||
|
||
async function checkLogo(logoUrl) {
|
||
if (!logoUrl) return false;
|
||
if (logoUrl.startsWith("/logos/") || logoUrl.startsWith("data:")) return true;
|
||
try {
|
||
const res = await withRetry(async () => {
|
||
return await fetch(logoUrl, { method: "HEAD", signal: AbortSignal.timeout(5000) });
|
||
}, { maxRetries: 2, label: `Logo ${logoUrl}` });
|
||
return res.ok;
|
||
} catch { return false; }
|
||
}
|
||
|
||
async function main() {
|
||
if (!process.env.DEEPSEEK_API_KEY) throw new Error("缺少环境变量: DEEPSEEK_API_KEY");
|
||
if (!process.env.DATABASE_URL) throw new Error("缺少环境变量: DATABASE_URL");
|
||
|
||
const startTime = new Date();
|
||
console.log(`🔍 [Task1] ${startTime.toISOString().split("T")[0]} 发现新AI工具...`);
|
||
|
||
const rssSources = [
|
||
{ url: "https://36kr.com/feed", name: "36kr" },
|
||
{ url: "https://www.ithome.com/rss/", name: "IT之家" },
|
||
{ url: "https://www.oschina.net/news/rss", name: "开源中国" },
|
||
];
|
||
|
||
let totalNews = 0;
|
||
let foundTools = 0;
|
||
let importedTools = 0;
|
||
const errors = [];
|
||
|
||
const existingNames = new Set([
|
||
...(await prisma.tool.findMany({ select: { name: true } })).map((t) => t.name.toLowerCase()),
|
||
...(await prisma.pendingTool.findMany({ select: { name: true } })).map((t) => t.name.toLowerCase()),
|
||
]);
|
||
|
||
const existingUrls = new Set([
|
||
...(await prisma.tool.findMany({ select: { websiteUrl: true } })).map((t) => t.websiteUrl || ""),
|
||
...(await prisma.pendingTool.findMany({ select: { websiteUrl: true } })).map((t) => t.websiteUrl || ""),
|
||
]);
|
||
|
||
// ---- 预筛：只对"标题即宣告新 AI 工具/产品"的新闻调用 AI，避免逐条调用（40 次/天 → ≤15 次/天）----
|
||
const NEG_TITLE_RE = /财报|股价|裁员|诉讼|处罚|召回|人事任命|评级|监管|关税|事故|禁令|上市申请|营收|亏损|融资|涨停|跌停|关停|下线|停运|放弃|炒作|愚弄|辟谣|传言/;
|
||
const ACTION_RE = /发布|上线|推出|开源|开放|内测|公测|首发|新增|更新|升级|亮相|登场|宣布/;
|
||
// 强 AI 主体词：命中即认为"可能是新 AI 工具/产品"（\bai\b 兼顾 "AI工具" 且不误命中 email/available）
|
||
const AI_STRONG_RE = /\bai\b|大模型|多模态|生成式|智能体|\bagent|copilot|\bllm\b|\bgpt\b|chatgpt|openai|anthropic|deepseek|claude|gemini|midjourney|\bsora\b|豆包|通义|文心|kimi|即梦|可灵|扣子|智谱/;
|
||
// 泛 AI 词：单独出现不足以判定，需标题配套动作词
|
||
const AI_WEAK_RE = /模型|助手|生成|对话|语音|图像|视频|编程|插件|工具|平台|\bapi\b|算法|推理/;
|
||
// 开源项目版本更新（如 "KnowForge 2026.0.5 发布"）不是"新 AI 工具"，prompt 已明确排除
|
||
const CHANGELOG_RE = /\d+\.\d+/;
|
||
const likelyTool = (t) => {
|
||
const title = (t.title || "").trim();
|
||
const titleLow = title.toLowerCase();
|
||
if (NEG_TITLE_RE.test(title) && !ACTION_RE.test(title)) return false; // 财经/负面类
|
||
if (CHANGELOG_RE.test(title) && /发布|更新|新增|升级/.test(title) && !AI_STRONG_RE.test(titleLow)) return false; // 版本更新
|
||
if (AI_STRONG_RE.test(titleLow)) return true; // 标题含强 AI 主体词
|
||
return ACTION_RE.test(title) && AI_WEAK_RE.test(titleLow); // 标题含"动作词 + 泛 AI 词"
|
||
};
|
||
const MAX_AI_CALLS = 15; // 单次运行 AI 调用上限（成本保护）
|
||
let aiCalls = 0;
|
||
let prefiltered = 0;
|
||
|
||
const categoryMap = {};
|
||
const categories = await prisma.category.findMany({ select: { id: true, name: true, slug: true } });
|
||
for (const c of categories) {
|
||
const name = c.name.toLowerCase();
|
||
const slugLower = c.slug.toLowerCase();
|
||
if (name.includes("文本") || slugLower.includes("text")) categoryMap["文本生成"] = c.id;
|
||
else if (name.includes("图像") || slugLower.includes("image")) categoryMap["图像生成"] = c.id;
|
||
else if (name.includes("视频") || slugLower.includes("video")) categoryMap["视频生成"] = c.id;
|
||
else if (name.includes("语音") || name.includes("音频") || slugLower.includes("audio")) categoryMap["语音音频"] = c.id;
|
||
else if (name.includes("编程") || name.includes("代码") || slugLower.includes("code")) categoryMap["代码编程"] = c.id;
|
||
else if (name.includes("办公") || slugLower.includes("office")) categoryMap["办公效率"] = c.id;
|
||
else if (name.includes("营销") || slugLower.includes("marketing")) categoryMap["营销创作"] = c.id;
|
||
else if (name.includes("数据") || name.includes("AI") || slugLower.includes("data")) categoryMap["数据AI"] = c.id;
|
||
else if (name.includes("教育") || slugLower.includes("education")) categoryMap["教育学习"] = c.id;
|
||
else if (name.includes("设计") || slugLower.includes("design")) categoryMap["设计制作"] = c.id;
|
||
}
|
||
|
||
for (const src of rssSources) {
|
||
console.log(` 📡 ${src.name}...`);
|
||
const news = await fetchRSS(src.url);
|
||
totalNews += news.length;
|
||
console.log(` 获取 ${news.length} 条`);
|
||
|
||
for (const item of news) {
|
||
// 预筛：非"疑似新工具"的新闻直接跳过，不消耗 AI 调用
|
||
if (!likelyTool(item) || aiCalls >= MAX_AI_CALLS) { prefiltered++; continue; }
|
||
const text = `${item.title} ${item.content}`;
|
||
try {
|
||
aiCalls++;
|
||
const completion = await withRetry(async () => {
|
||
return await openai.chat.completions.create({
|
||
model: process.env.DEEPSEEK_MODEL || "deepseek-v4-pro",
|
||
messages: [{ role: "user", content: EXTRACT_PROMPT.replace("{content}", text.substring(0, 1500)) }],
|
||
temperature: 0.2,
|
||
}, { signal: AbortSignal.timeout(120000) });
|
||
}, { maxRetries: 3, label: "DeepSeek API" });
|
||
const content = completion.choices[0]?.message?.content;
|
||
if (!content) continue;
|
||
|
||
let tools;
|
||
try { tools = extractJSON(content); } catch { continue; }
|
||
if (!Array.isArray(tools)) continue;
|
||
foundTools += tools.length;
|
||
|
||
for (const tool of tools) {
|
||
if (!tool.name || !tool.websiteUrl) continue;
|
||
if (existingNames.has(tool.name.toLowerCase()) || existingUrls.has(tool.websiteUrl)) continue;
|
||
|
||
const websiteOk = await checkWebsite(tool.websiteUrl);
|
||
const logoOk = await checkLogo(tool.logoUrl || "");
|
||
const catId = categoryMap[tool.category] || categoryMap["数据AI"] || categories[0]?.id || 1;
|
||
|
||
await prisma.pendingTool.create({
|
||
data: {
|
||
name: tool.name,
|
||
description: tool.description || "",
|
||
websiteUrl: tool.websiteUrl,
|
||
logoUrl: tool.logoUrl,
|
||
categoryId: catId,
|
||
sourceUrl: "AI自动发现",
|
||
status: "pending",
|
||
checkWebsiteOk: websiteOk,
|
||
checkLogoOk: logoOk,
|
||
checkDescOk: true,
|
||
checkCategoryOk: true,
|
||
checkDetail: { source: src.name, extractedAt: new Date().toISOString() },
|
||
},
|
||
});
|
||
|
||
existingNames.add(tool.name.toLowerCase());
|
||
existingUrls.add(tool.websiteUrl);
|
||
importedTools++;
|
||
console.log(` ✅ ${tool.name} (网站:${websiteOk ? "OK" : "FAIL"}, Logo:${logoOk ? "OK" : "FAIL"})`);
|
||
}
|
||
} catch (e) {
|
||
errors.push(`${src.name}/${item.title}: ${e.message}`);
|
||
}
|
||
await new Promise((r) => setTimeout(r, 300));
|
||
}
|
||
}
|
||
|
||
const endTime = new Date();
|
||
const duration = endTime.getTime() - startTime.getTime();
|
||
|
||
const result = {
|
||
totalNews,
|
||
prefiltered,
|
||
aiCalls,
|
||
foundTools,
|
||
importedTools,
|
||
duration: `${(duration / 1000).toFixed(1)}s`,
|
||
};
|
||
console.log(`\n🏁 [Task1] 完成: 新闻${totalNews}条, 预筛跳过${prefiltered}条, AI调用${aiCalls}次, 发现${foundTools}个, 入库${importedTools}个, 耗时${result.duration}`);
|
||
if (errors.length > 0) console.log(` 错误: ${errors.slice(0, 3).join("; ")}`);
|
||
|
||
await prisma.taskLog.create({
|
||
data: {
|
||
taskKey: "task1-discover-tools",
|
||
taskName: "发现AI工具",
|
||
status: errors.length > 0 && importedTools === 0 ? "failed" : "completed",
|
||
startedAt: startTime,
|
||
finishedAt: endTime,
|
||
duration,
|
||
result,
|
||
error: errors.length > 0 ? errors.slice(0, 5).join("\n") : null,
|
||
},
|
||
});
|
||
|
||
await prisma.$disconnect();
|
||
}
|
||
|
||
main()
|
||
.then(() => { console.log("Done"); process.exit(0); })
|
||
.catch(async (e) => {
|
||
console.error("❌ 任务一失败:", e.message);
|
||
if (prisma) {
|
||
try {
|
||
await prisma.taskLog.create({
|
||
data: {
|
||
taskKey: "task1-discover-tools",
|
||
taskName: "发现AI工具",
|
||
status: "failed",
|
||
startedAt: new Date(),
|
||
finishedAt: new Date(),
|
||
duration: 0,
|
||
error: e.message,
|
||
},
|
||
});
|
||
} catch {}
|
||
}
|
||
process.exit(1);
|
||
}).finally(() => prisma.$disconnect()); |