import { PrismaClient } from "@prisma/client"; import { PrismaMariaDb } from "@prisma/adapter-mariadb"; import OpenAI from "openai"; import "dotenv/config"; import { withRetry } from "./lib/retry.mjs"; import { cacheGet, cacheSet, disconnectRedis } from "./lib/redis-cache.mjs"; const rawUrl = process.env.DATABASE_URL; if (!rawUrl) { console.error("DATABASE_URL not set"); process.exit(1); } const base = rawUrl.replace("mysql://", "mariadb://"); const sep = base.includes("?") ? "&" : "?"; const connectionString = `${base}${sep}connection_limit=20&pool_timeout=30`; const adapter = new PrismaMariaDb(connectionString); const prisma = new PrismaClient({ adapter }); const openai = new OpenAI({ apiKey: process.env.DEEPSEEK_API_KEY, baseURL: "https://api.deepseek.com", timeout: 120000, maxRetries: 2, }); async function fetchRSS(rssUrl, source) { // 缓存 RSS 结果 30 分钟,避免重复请求 const cacheKey = `rss:${source}:${rssUrl}`; const cached = await cacheGet(cacheKey); if (cached) { console.log(` 📦 ${source} RSS 命中缓存 (${cached.length} 条)`); return cached; } try { const response = await fetch(rssUrl, { headers: { "User-Agent": "Mozilla/5.0 (compatible; NewsBot/1.0)" }, signal: AbortSignal.timeout(10000), }); if (!response.ok) return []; const xml = await response.text(); const items = []; const itemRegex = /([\s\S]*?)<\/item>/g; let match; while ((match = itemRegex.exec(xml)) !== null) { const block = match[1]; const titleMatch = block.match(/<!\[CDATA\[([\s\S]*?)\]\]><\/title>/) || block.match(/<title>([\s\S]*?)<\/title>/); const linkMatch = block.match(/<link>([\s\S]*?)<\/link>/); const descMatch = block.match(/<description><!\[CDATA\[([\s\S]*?)\]\]><\/description>/) || block.match(/<description>([\s\S]*?)<\/description>/); const pubDateMatch = block.match(/<pubDate>([\s\S]*?)<\/pubDate>/) || block.match(/<published>([\s\S]*?)<\/published>/); if (titleMatch) { items.push({ title: titleMatch[1].trim(), url: linkMatch ? linkMatch[1].trim() : "", content: descMatch ? descMatch[1].replace(/<[^>]+>/g, "").trim() : "", pubDate: pubDateMatch ? pubDateMatch[1].trim() : "", source, }); } } // 缓存 30 分钟 if (items.length > 0) { await cacheSet(cacheKey, items, 1800); } return items; } catch (e) { console.error(` ⚠️ ${source} RSS获取失败: ${e.message}`); return []; } } async function fetch36krAI() { return fetchRSS("https://36kr.com/feed", "36kr"); } async function fetchITHomeAI() { return fetchRSS("https://www.ithome.com/rss/", "IT之家"); } async function fetchOSChinaAI() { return fetchRSS("https://www.oschina.net/news/rss", "开源中国"); } async function fetchGoogleNewsRSS(query, lang = "zh-CN", country = "CN") { const url = `https://news.google.com/rss/search?q=${encodeURIComponent(query)}&hl=${lang}&gl=${country}&ceid=${country}:${lang === "zh-CN" ? "zh-Hans" : "en"}`; const results = await fetchRSS(url, `GoogleNews-${lang}`); return results; } async function fetchHackerNewsTop() { try { const topRes = await fetch("https://hacker-news.firebaseio.com/v0/topstories.json"); const topIds = await topRes.json(); const aiIds = topIds.slice(0, 60); const items = []; for (const id of aiIds) { try { const res = await fetch(`https://hacker-news.firebaseio.com/v0/item/${id}.json`); const story = await res.json(); if (!story || !story.title) continue; const title = story.title.toLowerCase(); if ( title.includes("ai") || title.includes("llm") || title.includes("gpt") || title.includes("openai") || title.includes("deepseek") || title.includes("machine learning") || title.includes("neural") || title.includes("model") || title.includes("claude") || title.includes("gemini") || title.includes("agent") || title.includes("rag") || title.includes("transformer") ) { items.push({ title: story.title, url: story.url || `https://news.ycombinator.com/item?id=${story.id}`, content: "", pubDate: new Date(story.time * 1000).toISOString(), }); } if (items.length >= 10) break; } catch {} } return items; } catch (e) { console.error(` ⚠️ HackerNews获取失败: ${e.message}`); return []; } } async function searchAllNews() { console.log("🔍 第一步:搜索真实AI新闻..."); const allResults = []; const seenTitles = new Set(); function addResults(results, source) { for (const r of results) { const key = r.title.substring(0, 30).toLowerCase(); if (!seenTitles.has(key) && r.title) { seenTitles.add(key); allResults.push({ ...r, source }); } } } console.log(" 📡 36kr..."); const kr36 = await fetch36krAI(); addResults(kr36, "36kr"); console.log(` 获取 ${kr36.length} 条`); console.log(" 📡 IT之家..."); const ithome = await fetchITHomeAI(); addResults(ithome, "IT之家"); console.log(` 获取 ${ithome.length} 条`); console.log(" 📡 开源中国..."); const oschina = await fetchOSChinaAI(); addResults(oschina, "开源中国"); console.log(` 获取 ${oschina.length} 条`); console.log(" 📡 Google News (中文)..."); const zhNews = await fetchGoogleNewsRSS("人工智能 AI 最新", "zh-CN", "CN"); addResults(zhNews, "GoogleNews-CN"); console.log(` 获取 ${zhNews.length} 条`); console.log(" 📡 Google News (English)..."); const enNews = await fetchGoogleNewsRSS("artificial intelligence AI latest news", "en", "US"); addResults(enNews, "GoogleNews-EN"); console.log(` 获取 ${enNews.length} 条`); console.log(" 📡 HackerNews Top AI..."); const hnNews = await fetchHackerNewsTop(); addResults(hnNews, "HackerNews"); console.log(` 获取 ${hnNews.length} 条`); const AI_KEYWORDS = [ "ai", "人工智能", "大模型", "llm", "gpt", "chatgpt", "openai", "deepseek", "claude", "gemini", "通义", "文心", "智谱", "kimi", "豆包", "百川", "diffusion", "stable diffusion", "midjourney", "sora", "机器学习", "深度学习", "神经网络", "transformer", "copilot", "agent", "rag", "aigc", "生成式", "自动驾驶", "智能体", "多模态", "语音识别", ]; function isAIRelated(title) { const lower = title.toLowerCase(); return AI_KEYWORDS.some((kw) => lower.includes(kw)); } const recentResults = allResults .filter((r) => { if (!isAIRelated(r.title)) return false; if (!r.pubDate) return true; try { const pubTime = new Date(r.pubDate).getTime(); const twoDaysAgo = Date.now() - 2 * 24 * 60 * 60 * 1000; return pubTime > twoDaysAgo; } catch { return true; } }); const finalResults = recentResults.length > 0 ? recentResults : allResults.slice(0, 30); console.log(` ✅ 共获取 ${allResults.length} 条,近2天 ${recentResults.length} 条,使用 ${finalResults.length} 条`); return finalResults; } const FORMAT_PROMPT = `你是AI行业资深分析师。以下是从互联网搜索到的今日真实AI新闻素材,请基于这些真实素材整理一份AI日报。 严格要求: 1. 只能基于提供的素材内容,绝对不能编造任何新闻!如果素材不够6条,就只输出有素材支持的条目 2. 每条新闻必须保留原始来源URL 3. 标题简洁有力(20字以内),摘要精炼(50字以内),详情深入(100-200字) 4. 优先关注中国AI行业动态,兼顾全球重要事件 5. 分类标注规则(必须使用以下key值): - model:模型前沿(LLM/多模态新模型发布、架构创新、训练方法突破) - product:产品落地(AI产品发布/更新、C端应用上线、B端解决方案、新功能) - opensource:开源生态(开源项目动态、框架更新、开源社区大事件) - business:商业产业(投融资、收购并购、财报营收、公司战略、行业趋势) - policy:政策治理(AI法规政策、监管动态、伦理治理、国际AI治理) - insight:深度洞察(行业领袖观点、深度分析报告、学术研究前沿) 新闻素材: {newsData} 返回纯JSON格式: { "title": "AI日报:{date} 核心要点概览", "summary": "今日AI行业整体趋势概述(100字以内)", "items": [ { "category": "model", "title": "新闻标题", "summary": "一句话摘要", "detail": "详细内容描述", "sourceUrl": "原始来源URL", "hotScore": 85 } ] } hotScore热度评分规则(0-100): - 90-100:全球头条级事件 - 70-89:行业重要事件 - 50-69:值得关注的动态 - 30-49:一般性新闻`; function extractJSON(str) { const trimmed = str.trim(); const codeBlockMatch = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/); let jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : trimmed; const braceMatch = jsonStr.match(/\{[\s\S]*\}/); if (braceMatch) { try { return JSON.parse(braceMatch[0]); } catch {} } const arrayMatch = jsonStr.match(/\[[\s\S]*\]/); if (arrayMatch) { try { return JSON.parse(arrayMatch[0]); } catch {} } return JSON.parse(jsonStr); } async function generateDailyNews() { const now = new Date(); const cnOffset = 8 * 60 * 60 * 1000; const cnTime = new Date(now.getTime() + cnOffset); const dateStr = cnTime.toLocaleDateString("zh-CN", { year: "numeric", month: "long", day: "numeric", timeZone: "Asia/Shanghai", }); const dateISO = cnTime.toISOString().split("T")[0]; console.log(`📰 开始生成 ${dateStr} 的AI日报...`); const existing = await prisma.dailyReport.findFirst({ where: { date: new Date(dateISO) }, }); if (existing) { console.log(`⚠️ ${dateISO} 已有日报 (ID:${existing.id}),先删除旧版`); await prisma.dailyReport.delete({ where: { id: existing.id } }); } const newsResults = await searchAllNews(); if (newsResults.length === 0) { throw new Error("未搜索到任何新闻素材,无法生成日报"); } const newsDataText = newsResults .slice(0, 14) .map((r, i) => `[${i + 1}] 标题: ${r.title}\n 内容: ${r.content || "(无摘要)"}\n 来源: ${r.url}\n 时间: ${r.pubDate || "未知"}`) .join("\n\n"); console.log("🤖 第二步:调用 DeepSeek 整理新闻格式..."); const prompt = FORMAT_PROMPT .replace(/\{newsData\}/g, newsDataText) .replace(/\{date\}/g, dateStr); const completion = await withRetry(async () => { return await openai.chat.completions.create({ model: process.env.DEEPSEEK_MODEL || "deepseek-v4-pro", messages: [{ role: "user", content: prompt }], temperature: 0.3, // DeepSeek V4 reasoning token 计入 max_tokens 预算:预算不足会导致 content 为空(finish_reason=length) max_tokens: 16384, }, { signal: AbortSignal.timeout(180000) }); }, { maxRetries: 3, label: "DeepSeek News API" }); const u = completion.usage; if (u) { console.log(` ℹ️ tokens: prompt=${u.prompt_tokens} completion=${u.completion_tokens} reasoning=${u.completion_tokens_details?.reasoning_tokens ?? "n/a"} finish=${completion.choices[0]?.finish_reason}`); } const content = completion.choices[0]?.message?.content; if (!content) { throw new Error(`DeepSeek 返回内容为空(finish_reason=${completion.choices[0]?.finish_reason},completion_tokens=${u?.completion_tokens},疑似 reasoning 占满 max_tokens)`); } let parsed; try { parsed = extractJSON(content); } catch (e) { console.error("❌ JSON解析失败,原始内容前500字:"); console.error(content.substring(0, 500)); throw e; } if (!parsed.items || !Array.isArray(parsed.items) || parsed.items.length === 0) { throw new Error("整理后的日报没有有效的新闻条目"); } console.log(`✅ 整理成功,共 ${parsed.items.length} 条新闻`); const report = await prisma.dailyReport.create({ data: { date: new Date(dateISO), title: parsed.title || `AI日报:${dateStr} 核心要点概览`, summary: parsed.summary || "", hotCount: parsed.items.length, status: "published", items: { create: parsed.items.map((item, index) => ({ sortOrder: index, category: item.category || "product", title: item.title || "未命名新闻", summary: item.summary || "", detail: item.detail || "", sourceUrl: item.sourceUrl || "", imageUrl: item.imageUrl || "", hotScore: item.hotScore || 50, })), }, }, include: { items: true }, }); console.log(`\n🎉 日报生成完成!`); console.log(` ID: ${report.id}`); console.log(` 标题: ${report.title}`); console.log(` 日期: ${dateISO}`); console.log(` 条目数: ${report.items.length}`); console.log(` 状态: ${report.status}`); report.items.forEach((item, i) => { console.log(` ${i + 1}. [${item.category}] ${item.title} (热度:${item.hotScore})`); if (item.sourceUrl) { console.log(` 来源: ${item.sourceUrl}`); } }); return report; } async function generateWithRetry() { for (let attempt = 0; attempt < 2; attempt++) { try { return await generateDailyNews(); } catch (e) { const msg = String(e.message || e); // 偶发中止/超时/上游 5xx 才重试;业务类错误直接抛出 const retryable = /abort|timeout|ETIMEDOUT|ECONNRESET|socket|fetch failed|502|503|429/i.test(msg); if (!retryable || attempt === 1) throw e; console.warn(`⚠️ 日报生成中止(${msg.slice(0, 80)}),30s 后重试 (${attempt + 1}/2)...`); await new Promise((r) => setTimeout(r, 30000)); } } throw new Error("unreachable"); } async function main() { if (!process.env.DEEPSEEK_API_KEY) throw new Error("缺少环境变量: DEEPSEEK_API_KEY"); if (!process.env.DATABASE_URL) throw new Error("缺少环境变量: DATABASE_URL"); const startTime = new Date(); try { const report = await generateWithRetry(); const endTime = new Date(); const duration = endTime.getTime() - startTime.getTime(); await prisma.taskLog.create({ data: { taskKey: "daily-news", taskName: "AI日报生成", status: "completed", startedAt: startTime, finishedAt: endTime, duration, result: { reportId: report.id, itemCount: report.items.length, title: report.title }, }, }); } catch (error) { console.error("❌ 日报生成失败:", error.message); try { await prisma.taskLog.create({ data: { taskKey: "daily-news", taskName: "AI日报生成", status: "failed", startedAt: startTime, finishedAt: new Date(), duration: new Date().getTime() - startTime.getTime(), error: error.message, }, }); } catch {} process.exit(1); } finally { await prisma.$disconnect(); await disconnectRedis(); } } main() .then(() => { console.log("Done"); process.exit(0); }) .catch((e) => { console.error(e); process.exit(1); }) .finally(() => prisma.$disconnect());