zhuiguang-ai/scripts/trend-alert.mjs

/**
 * 行业趋势监测脚本 — 每日 10:00 运行
 * 从 HackerNews API 拉取热帖，筛选 AI 相关主题
 */
import "dotenv/config";
import fs from "fs";
import path from "path";
import { fileURLToPath } from "url";

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const DATA_DIR = path.join(__dirname, "..", "data", "trend-alerts");

const HN_TOP_STORIES = "https://hacker-news.firebaseio.com/v0/topstories.json";
const HN_ITEM = "https://hacker-news.firebaseio.com/v0/item";
// 主数据源：Algolia HN API（全球可达、单请求返回热帖含分数/评论，避免 firebaseio 从国内服务器超时）
const HN_ALGOLIA = "https://hn.algolia.com/api/v1/search?tags=front_page&hitsPerPage=100";

// AI 相关关键词
const AI_KEYWORDS = [
  "ai", "llm", "gpt", "machine learning", "deep learning",
  "openai", "anthropic", "claude", "gemini", "copilot",
  "agent", "prompt", "rag", "vector", "embedding",
  "transformer", "LLM", "GPT", "AI", "chatgpt", "ChatGPT",
  "diffusion", "stable diffusion", "langchain", "fine-tun",
  "neural", "inference", "open source ai", "llama", "mistral",
];

function todayDateStr() {
  return new Date().toISOString().slice(0, 10);
}

function log(message) {
  const ts = new Date().toISOString();
  console.log(`[${ts}] ${message}`);
}

async function fetchJson(url) {
  let lastErr;
  for (let attempt = 0; attempt < 3; attempt++) {
    try {
      const res = await fetch(url, { signal: AbortSignal.timeout(12000) });
      if (!res.ok) throw new Error(`HTTP ${res.status}`);
      return await res.json();
    } catch (e) {
      lastErr = e;
      if (attempt < 2) await new Promise((r) => setTimeout(r, 500 * (attempt + 1)));
    }
  }
  throw lastErr;
}

function matchesAIKeywords(title) {
  if (!title) return false;
  const lowerTitle = title.toLowerCase();
  for (const keyword of AI_KEYWORDS) {
    if (lowerTitle.includes(keyword.toLowerCase())) {
      return true;
    }
  }
  return false;
}

async function main() {
  log("行业趋势监测开始...");

  // 确保数据目录存在
  if (!fs.existsSync(DATA_DIR)) {
    fs.mkdirSync(DATA_DIR, { recursive: true });
  }

  // 主源：Algolia（单请求、全球可达）；失败回退 firebaseio 两段式拉取
  let items = [];
  let source = "algolia";
  try {
    items = await fetchAlgoliaTop();
    log(`[algolia] 获取到 ${items.length} 条热帖`);
  } catch (e) {
    source = "firebase";
    log(`[algolia] 失败(${e.message})，回退 firebaseio...`);
    items = await fetchFirebaseTop();
    log(`[firebase] 获取到 ${items.length} 条热帖`);
  }

  // 筛选 AI 相关（字段已统一：title/url/score/commentCount/by/time）
  const aiItems = items
    .filter((item) => matchesAIKeywords(item.title))
    .sort((a, b) => (b.score || 0) - (a.score || 0))
    .slice(0, 5);

  log(`筛选出 ${aiItems.length} 条 AI 相关热帖`);

  const dateStr = todayDateStr();
  const alertFile = path.join(DATA_DIR, `${dateStr}.json`);

  const alert = {
    date: dateStr,
    generatedAt: new Date().toISOString(),
    source,
    items: aiItems,
    summary: `从 ${items.length} 条热帖中筛选出 ${aiItems.length} 条 AI 相关话题`,
  };

  fs.writeFileSync(alertFile, JSON.stringify(alert, null, 2), "utf-8");
  log(`趋势报告已保存: ${alertFile}`);

  for (const item of aiItems) {
    log(`  🏷 ${item.title} (${item.score} pts, ${item.commentCount} comments)`);
  }

  log("行业趋势监测完成");
}

async function fetchAlgoliaTop() {
  const data = await fetchJson(HN_ALGOLIA);
  return (data.hits || [])
    .filter((h) => h.title)
    .map((h) => ({
      id: h.objectID,
      title: h.title,
      url: h.url || `https://news.ycombinator.com/item?id=${h.objectID}`,
      score: h.points || 0,
      commentCount: h.num_comments || 0,
      by: h.author || "",
      time: h.created_at || "",
    }));
}

async function fetchFirebaseTop() {
  const storyIds = await fetchJson(HN_TOP_STORIES);
  log(`[firebase] 获取到 ${storyIds.length} 条热帖 ID`);
  const top100Ids = storyIds.slice(0, 100);
  const items = [];
  for (let i = 0; i < top100Ids.length; i += 10) {
    const batch = top100Ids.slice(i, i + 10);
    const results = await Promise.all(
      batch.map((id) => fetchJson(`${HN_ITEM}/${id}.json`).catch(() => null))
    );
    for (const item of results) {
      if (!item) continue;
      items.push({
        id: item.id,
        title: item.title,
        url: item.url || `https://news.ycombinator.com/item?id=${item.id}`,
        score: item.score || 0,
        commentCount: item.descendants || 0,
        by: item.by || "",
        time: item.time ? new Date(item.time * 1000).toISOString() : "",
      });
    }
    if (i + 10 < top100Ids.length) {
      await new Promise((r) => setTimeout(r, 200));
    }
  }
  return items;
}

main().catch((err) => {
  console.error("行业趋势监测失败:", err);
  process.exit(1);
});