231 lines
9.2 KiB
JavaScript
231 lines
9.2 KiB
JavaScript
import { PrismaClient } from "@prisma/client";
|
|
import { PrismaMariaDb } from "@prisma/adapter-mariadb";
|
|
import OpenAI from "openai";
|
|
import "dotenv/config";
|
|
import { withRetry } from "./lib/retry.mjs";
|
|
|
|
const base = process.env.DATABASE_URL || "mysql://localhost:3306/zhuiguang_ai?charset=utf8mb4";
|
|
const sep = base.includes("?") ? "&" : "?";
|
|
const connectionString = `${base}${sep}connection_limit=20&pool_timeout=30`;
|
|
const adapter = new PrismaMariaDb(connectionString);
|
|
const prisma = new PrismaClient({ adapter });
|
|
const openai = new OpenAI({ apiKey: process.env.DEEPSEEK_API_KEY, baseURL: "https://api.deepseek.com/v1" });
|
|
|
|
const EXTRACT_PROMPT = `你是一个AI工具信息提取助手。从以下科技新闻内容中，提取出现的AI工具/产品信息。只提取明确提到的AI软件工具、AI平台、AI产品，不要提取开源项目库（GitHub仓库）、不要提取硬件、芯片。每个工具返回以下信息（如果信息不足则跳过该工具）：
|
|
|
|
新闻内容：
|
|
{content}
|
|
|
|
返回纯JSON数组：
|
|
[{
|
|
"name": "工具名称",
|
|
"description": "一句话简介（20字以内）",
|
|
"websiteUrl": "官网或产品主页URL",
|
|
"category": "分类关键词（只能从以下选一）：文本生成/图像生成/视频生成/语音音频/代码编程/办公效率/营销创作/数据AI/教育学习/设计制作"
|
|
}]`;
|
|
|
|
function extractJSON(str) {
|
|
const trimmed = str.trim();
|
|
const codeBlockMatch = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/);
|
|
let jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : trimmed;
|
|
const arrayMatch = jsonStr.match(/\[[\s\S]*\]/);
|
|
if (arrayMatch) {
|
|
try { return JSON.parse(arrayMatch[0]); } catch {}
|
|
}
|
|
return JSON.parse(jsonStr);
|
|
}
|
|
|
|
async function fetchRSS(url) {
|
|
try {
|
|
const res = await withRetry(async () => {
|
|
return await fetch(url, {
|
|
headers: { "User-Agent": "Mozilla/5.0 (compatible; ToolBot/1.0)" },
|
|
signal: AbortSignal.timeout(10000),
|
|
});
|
|
}, { maxRetries: 3, label: `RSS ${url}` });
|
|
if (!res.ok) return [];
|
|
const xml = await res.text();
|
|
const items = [];
|
|
const regex = /<item>([\s\S]*?)<\/item>/g;
|
|
let m;
|
|
while ((m = regex.exec(xml)) !== null) {
|
|
const b = m[1];
|
|
const t = (b.match(/<title><!\[CDATA\[([\s\S]*?)\]\]><\/title>/) || b.match(/<title>([\s\S]*?)<\/title>/))?.[1] || "";
|
|
const d = (b.match(/<description><!\[CDATA\[([\s\S]*?)\]\]><\/description>/) || b.match(/<description>([\s\S]*?)<\/description>/))?.[1] || "";
|
|
items.push({ title: t.trim(), content: d.replace(/<[^>]+>/g, "").trim().substring(0, 500) });
|
|
}
|
|
return items.slice(0, 20);
|
|
} catch { return []; }
|
|
}
|
|
|
|
async function checkWebsite(url) {
|
|
try {
|
|
const res = await withRetry(async () => {
|
|
return await fetch(url, { method: "GET", signal: AbortSignal.timeout(8000), redirect: "follow" });
|
|
}, { maxRetries: 2, label: `Website ${url}` });
|
|
return res.ok;
|
|
} catch { return false; }
|
|
}
|
|
|
|
async function checkLogo(logoUrl) {
|
|
if (!logoUrl) return false;
|
|
if (logoUrl.startsWith("/logos/") || logoUrl.startsWith("data:")) return true;
|
|
try {
|
|
const res = await withRetry(async () => {
|
|
return await fetch(logoUrl, { method: "HEAD", signal: AbortSignal.timeout(5000) });
|
|
}, { maxRetries: 2, label: `Logo ${logoUrl}` });
|
|
return res.ok;
|
|
} catch { return false; }
|
|
}
|
|
|
|
async function main() {
|
|
const startTime = new Date();
|
|
console.log(`🔍 [Task1] ${startTime.toISOString().split("T")[0]} 发现新AI工具...`);
|
|
|
|
const rssSources = [
|
|
{ url: "https://36kr.com/feed", name: "36kr" },
|
|
{ url: "https://www.ithome.com/rss/", name: "IT之家" },
|
|
{ url: "https://www.oschina.net/news/rss", name: "开源中国" },
|
|
];
|
|
|
|
let totalNews = 0;
|
|
let foundTools = 0;
|
|
let importedTools = 0;
|
|
const errors: string[] = [];
|
|
|
|
const existingNames = new Set([
|
|
...(await prisma.tool.findMany({ select: { name: true } })).map((t) => t.name.toLowerCase()),
|
|
...(await prisma.pendingTool.findMany({ select: { name: true } })).map((t) => t.name.toLowerCase()),
|
|
]);
|
|
|
|
const existingUrls = new Set([
|
|
...(await prisma.tool.findMany({ select: { websiteUrl: true } })).map((t) => t.websiteUrl || ""),
|
|
...(await prisma.pendingTool.findMany({ select: { websiteUrl: true } })).map((t) => t.websiteUrl || ""),
|
|
]);
|
|
|
|
const categoryMap: Record<string, number> = {};
|
|
const categories = await prisma.category.findMany({ select: { id: true, name: true, slug: true } });
|
|
for (const c of categories) {
|
|
const name = c.name.toLowerCase();
|
|
const slugLower = c.slug.toLowerCase();
|
|
if (name.includes("文本") || slugLower.includes("text")) categoryMap["文本生成"] = c.id;
|
|
else if (name.includes("图像") || slugLower.includes("image")) categoryMap["图像生成"] = c.id;
|
|
else if (name.includes("视频") || slugLower.includes("video")) categoryMap["视频生成"] = c.id;
|
|
else if (name.includes("语音") || name.includes("音频") || slugLower.includes("audio")) categoryMap["语音音频"] = c.id;
|
|
else if (name.includes("编程") || name.includes("代码") || slugLower.includes("code")) categoryMap["代码编程"] = c.id;
|
|
else if (name.includes("办公") || slugLower.includes("office")) categoryMap["办公效率"] = c.id;
|
|
else if (name.includes("营销") || slugLower.includes("marketing")) categoryMap["营销创作"] = c.id;
|
|
else if (name.includes("数据") || name.includes("AI") || slugLower.includes("data")) categoryMap["数据AI"] = c.id;
|
|
else if (name.includes("教育") || slugLower.includes("education")) categoryMap["教育学习"] = c.id;
|
|
else if (name.includes("设计") || slugLower.includes("design")) categoryMap["设计制作"] = c.id;
|
|
}
|
|
|
|
for (const src of rssSources) {
|
|
console.log(` 📡 ${src.name}...`);
|
|
const news = await fetchRSS(src.url);
|
|
totalNews += news.length;
|
|
console.log(` 获取 ${news.length} 条`);
|
|
|
|
for (const item of news) {
|
|
const text = `${item.title} ${item.content}`;
|
|
try {
|
|
const completion = await withRetry(async () => {
|
|
return await openai.chat.completions.create({
|
|
model: "deepseek-v4-pro",
|
|
messages: [{ role: "user", content: EXTRACT_PROMPT.replace("{content}", text.substring(0, 1500)) }],
|
|
temperature: 0.2,
|
|
});
|
|
}, { maxRetries: 3, label: "DeepSeek API" });
|
|
const content = completion.choices[0]?.message?.content;
|
|
if (!content) continue;
|
|
|
|
let tools;
|
|
try { tools = extractJSON(content); } catch { continue; }
|
|
if (!Array.isArray(tools)) continue;
|
|
foundTools += tools.length;
|
|
|
|
for (const tool of tools) {
|
|
if (!tool.name || !tool.websiteUrl) continue;
|
|
if (existingNames.has(tool.name.toLowerCase()) || existingUrls.has(tool.websiteUrl)) continue;
|
|
|
|
const websiteOk = await checkWebsite(tool.websiteUrl);
|
|
const logoOk = await checkLogo(tool.logoUrl || "");
|
|
const catId = categoryMap[tool.category] || categoryMap["数据AI"] || categories[0]?.id || 1;
|
|
|
|
await prisma.pendingTool.create({
|
|
data: {
|
|
name: tool.name,
|
|
description: tool.description || "",
|
|
websiteUrl: tool.websiteUrl,
|
|
logoUrl: tool.logoUrl,
|
|
categoryId: catId,
|
|
sourceUrl: "AI自动发现",
|
|
status: "pending",
|
|
checkWebsiteOk: websiteOk,
|
|
checkLogoOk: logoOk,
|
|
checkDescOk: true,
|
|
checkCategoryOk: true,
|
|
checkDetail: { source: src.name, extractedAt: new Date().toISOString() },
|
|
},
|
|
});
|
|
|
|
existingNames.add(tool.name.toLowerCase());
|
|
existingUrls.add(tool.websiteUrl);
|
|
importedTools++;
|
|
console.log(` ✅ ${tool.name} (网站:${websiteOk ? "OK" : "FAIL"}, Logo:${logoOk ? "OK" : "FAIL"})`);
|
|
}
|
|
} catch (e) {
|
|
errors.push(`${src.name}/${item.title}: ${e.message}`);
|
|
}
|
|
await new Promise((r) => setTimeout(r, 300));
|
|
}
|
|
}
|
|
|
|
const endTime = new Date();
|
|
const duration = endTime.getTime() - startTime.getTime();
|
|
|
|
const result = {
|
|
totalNews,
|
|
foundTools,
|
|
importedTools,
|
|
duration: `${(duration / 1000).toFixed(1)}s`,
|
|
};
|
|
console.log(`\n🏁 [Task1] 完成: 新闻${totalNews}条, 发现${foundTools}个, 入库${importedTools}个, 耗时${result.duration}`);
|
|
if (errors.length > 0) console.log(` 错误: ${errors.slice(0, 3).join("; ")}`);
|
|
|
|
await prisma.taskLog.create({
|
|
data: {
|
|
taskKey: "task1-discover-tools",
|
|
taskName: "发现AI工具",
|
|
status: errors.length > 0 && importedTools === 0 ? "failed" : "completed",
|
|
startedAt: startTime,
|
|
finishedAt: endTime,
|
|
duration,
|
|
result,
|
|
error: errors.length > 0 ? errors.slice(0, 5).join("\n") : null,
|
|
},
|
|
});
|
|
|
|
await prisma.$disconnect();
|
|
}
|
|
|
|
main().catch(async (e) => {
|
|
console.error("❌ 任务一失败:", e.message);
|
|
if (prisma) {
|
|
try {
|
|
await prisma.taskLog.create({
|
|
data: {
|
|
taskKey: "task1-discover-tools",
|
|
taskName: "发现AI工具",
|
|
status: "failed",
|
|
startedAt: new Date(),
|
|
finishedAt: new Date(),
|
|
duration: 0,
|
|
error: e.message,
|
|
},
|
|
});
|
|
} catch {}
|
|
await prisma.$disconnect();
|
|
}
|
|
process.exit(1);
|
|
}); |