/** * 行业趋势监测脚本 — 每日 10:00 运行 * 从 HackerNews API 拉取热帖,筛选 AI 相关主题 */ import "dotenv/config"; import fs from "fs"; import path from "path"; import { fileURLToPath } from "url"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const DATA_DIR = path.join(__dirname, "..", "data", "trend-alerts"); const HN_TOP_STORIES = "https://hacker-news.firebaseio.com/v0/topstories.json"; const HN_ITEM = "https://hacker-news.firebaseio.com/v0/item"; // 主数据源:Algolia HN API(全球可达、单请求返回热帖含分数/评论,避免 firebaseio 从国内服务器超时) const HN_ALGOLIA = "https://hn.algolia.com/api/v1/search?tags=front_page&hitsPerPage=100"; // AI 相关关键词 const AI_KEYWORDS = [ "ai", "llm", "gpt", "machine learning", "deep learning", "openai", "anthropic", "claude", "gemini", "copilot", "agent", "prompt", "rag", "vector", "embedding", "transformer", "LLM", "GPT", "AI", "chatgpt", "ChatGPT", "diffusion", "stable diffusion", "langchain", "fine-tun", "neural", "inference", "open source ai", "llama", "mistral", ]; function todayDateStr() { return new Date().toISOString().slice(0, 10); } function log(message) { const ts = new Date().toISOString(); console.log(`[${ts}] ${message}`); } async function fetchJson(url) { let lastErr; for (let attempt = 0; attempt < 3; attempt++) { try { const res = await fetch(url, { signal: AbortSignal.timeout(12000) }); if (!res.ok) throw new Error(`HTTP ${res.status}`); return await res.json(); } catch (e) { lastErr = e; if (attempt < 2) await new Promise((r) => setTimeout(r, 500 * (attempt + 1))); } } throw lastErr; } function matchesAIKeywords(title) { if (!title) return false; const lowerTitle = title.toLowerCase(); for (const keyword of AI_KEYWORDS) { if (lowerTitle.includes(keyword.toLowerCase())) { return true; } } return false; } async function main() { log("行业趋势监测开始..."); // 确保数据目录存在 if (!fs.existsSync(DATA_DIR)) { fs.mkdirSync(DATA_DIR, { recursive: true }); } // 主源:Algolia(单请求、全球可达);失败回退 firebaseio 两段式拉取 let items = []; let source = "algolia"; try { items = await fetchAlgoliaTop(); log(`[algolia] 获取到 ${items.length} 条热帖`); } catch (e) { source = "firebase"; log(`[algolia] 失败(${e.message}),回退 firebaseio...`); items = await fetchFirebaseTop(); log(`[firebase] 获取到 ${items.length} 条热帖`); } // 筛选 AI 相关(字段已统一:title/url/score/commentCount/by/time) const aiItems = items .filter((item) => matchesAIKeywords(item.title)) .sort((a, b) => (b.score || 0) - (a.score || 0)) .slice(0, 5); log(`筛选出 ${aiItems.length} 条 AI 相关热帖`); const dateStr = todayDateStr(); const alertFile = path.join(DATA_DIR, `${dateStr}.json`); const alert = { date: dateStr, generatedAt: new Date().toISOString(), source, items: aiItems, summary: `从 ${items.length} 条热帖中筛选出 ${aiItems.length} 条 AI 相关话题`, }; fs.writeFileSync(alertFile, JSON.stringify(alert, null, 2), "utf-8"); log(`趋势报告已保存: ${alertFile}`); for (const item of aiItems) { log(` 🏷 ${item.title} (${item.score} pts, ${item.commentCount} comments)`); } log("行业趋势监测完成"); } async function fetchAlgoliaTop() { const data = await fetchJson(HN_ALGOLIA); return (data.hits || []) .filter((h) => h.title) .map((h) => ({ id: h.objectID, title: h.title, url: h.url || `https://news.ycombinator.com/item?id=${h.objectID}`, score: h.points || 0, commentCount: h.num_comments || 0, by: h.author || "", time: h.created_at || "", })); } async function fetchFirebaseTop() { const storyIds = await fetchJson(HN_TOP_STORIES); log(`[firebase] 获取到 ${storyIds.length} 条热帖 ID`); const top100Ids = storyIds.slice(0, 100); const items = []; for (let i = 0; i < top100Ids.length; i += 10) { const batch = top100Ids.slice(i, i + 10); const results = await Promise.all( batch.map((id) => fetchJson(`${HN_ITEM}/${id}.json`).catch(() => null)) ); for (const item of results) { if (!item) continue; items.push({ id: item.id, title: item.title, url: item.url || `https://news.ycombinator.com/item?id=${item.id}`, score: item.score || 0, commentCount: item.descendants || 0, by: item.by || "", time: item.time ? new Date(item.time * 1000).toISOString() : "", }); } if (i + 10 < top100Ids.length) { await new Promise((r) => setTimeout(r, 200)); } } return items; } main().catch((err) => { console.error("行业趋势监测失败:", err); process.exit(1); });