import { PrismaClient } from "@prisma/client"; import { PrismaMariaDb } from "@prisma/adapter-mariadb"; import "dotenv/config"; const rawUrl = process.env.DATABASE_URL; if (!rawUrl) { console.error("DATABASE_URL not set"); process.exit(1); } const base = rawUrl.replace("mysql://", "mariadb://"); const sep = base.includes("?") ? "&" : "?"; const connectionString = `${base}${sep}connection_limit=10&pool_timeout=15`; const adapter = new PrismaMariaDb(connectionString); const prisma = new PrismaClient({ adapter }); const USER_AGENT = "Mozilla/5.0 (compatible; EnrichBot/1.0; +https://www.zhuig.com)"; const FETCH_TIMEOUT_MS = 10000; const RATE_LIMIT_MS = 2000; async function delay(ms) { return new Promise((r) => setTimeout(r, ms)); } function resolveUrl(href, base) { try { return new URL(href, base).href; } catch { return null; } } async function safeFetch(url) { try { return await fetch(url, { method: "GET", signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), redirect: "follow", headers: { "User-Agent": USER_AGENT }, }); } catch { return null; } } async function fetchOgData(websiteUrl) { const result = { logo: null, description: null }; let url; try { url = new URL(websiteUrl); if (!["http:", "https:"].includes(url.protocol)) return result; } catch { return result; } const res = await safeFetch(url.href); if (!res || !res.ok) return result; let html; try { html = await res.text(); } catch { return result; } const ogImageMatch = html.match(/").replace(/"/g, '"').replace(/'/g, "'"); if (desc.trim().length >= 10) { result.description = desc.trim(); } } if (!result.logo) { const faviconMatch = html.match(/]*rel=["'](?:shortcut\s+)?icon["'][^>]*href=["']([^"']+)["']/i) || html.match(/]*href=["']([^"']+)["'][^>]*rel=["'](?:shortcut\s+)?icon["']/i); if (faviconMatch) { const favicon = resolveUrl(faviconMatch[1], url.href); if (favicon) { result.logo = favicon; } } } if (!result.logo) { const domain = url.hostname; const faviconGuesses = [`${url.protocol}//${domain}/favicon.ico`, `${url.protocol}//${domain}/apple-touch-icon.png`]; for (const guess of faviconGuesses) { const headRes = await safeFetch(guess); if (headRes && headRes.ok) { const ct = headRes.headers.get("content-type") || ""; if (ct.startsWith("image/") || ct.includes("icon") || guess.endsWith(".ico")) { result.logo = guess; break; } } } } return result; } async function tryClearbitLogo(domain) { try { const url = `https://logo.clearbit.com/${encodeURIComponent(domain)}`; const res = await fetch(url, { method: "HEAD", signal: AbortSignal.timeout(8000), headers: { "User-Agent": USER_AGENT }, }); if (res.ok) { const ct = res.headers.get("content-type") || ""; if (ct.startsWith("image/")) return url; } } catch { // Clearbit unavailable, expected } return null; } async function enrichTool(tool) { let domain; try { domain = new URL(tool.websiteUrl).hostname; } catch { return null; } const needsLogo = !tool.logoUrl; const needsDesc = !tool.description || tool.description.length < 100; if (!needsLogo && !needsDesc) return null; const updates = {}; const findings = []; const og = await fetchOgData(tool.websiteUrl); if (needsLogo) { if (og.logo) { updates.logoUrl = og.logo; findings.push("logo"); } else { const clearbit = await tryClearbitLogo(domain); if (clearbit) { updates.logoUrl = clearbit; findings.push("logo(clearbit)"); } } } if (needsDesc && og.description && og.description.length > (tool.description?.length || 0)) { updates.description = og.description.slice(0, 500); findings.push("desc"); } if (Object.keys(updates).length > 0) { await prisma.tool.update({ where: { id: tool.id }, data: updates }); } return findings; } async function main() { const startTime = new Date(); const tools = await prisma.tool.findMany({ where: { status: "published", websiteUrl: { not: "" } }, select: { id: true, name: true, slug: true, websiteUrl: true, logoUrl: true, description: true, }, }); const needingEnrichment = tools.filter((t) => { const needsLogo = !t.logoUrl; const needsDesc = !t.description || t.description.length < 100; return needsLogo || needsDesc; }); console.log(`Found ${tools.length} published tools, ${needingEnrichment.length} need enrichment\n`); let enriched = 0; let logoFound = 0; let descFound = 0; let failed = 0; const errors = []; for (let i = 0; i < needingEnrichment.length; i++) { const tool = needingEnrichment[i]; const idx = i + 1; const total = needingEnrichment.length; process.stdout.write(`[${idx}/${total}] ${tool.name}: `); try { const findings = await enrichTool(tool); if (findings && findings.length > 0) { enriched++; if (findings.includes("logo") || findings.includes("logo(clearbit)")) logoFound++; if (findings.includes("desc")) descFound++; process.stdout.write(`${findings.join(", ")} ✓\n`); } else { process.stdout.write(`nothing to enrich\n`); } } catch (err) { failed++; errors.push(`${tool.name}: ${err.message}`); process.stdout.write(`error: ${err.message}\n`); } if (idx < total) { await delay(RATE_LIMIT_MS); } } const endTime = new Date(); const duration = endTime.getTime() - startTime.getTime(); console.log(`\nDone.`); console.log(` Tools processed: ${needingEnrichment.length}`); console.log(` Enriched: ${enriched} (logo: ${logoFound}, description: ${descFound})`); console.log(` Failed: ${failed}`); console.log(` Duration: ${(duration / 1000).toFixed(1)}s`); if (errors.length > 0) { console.log(`\nErrors:`); errors.forEach((e) => console.log(` - ${e}`)); } await prisma.$disconnect(); } main().catch(async (err) => { console.error("Fatal:", err.message); if (prisma) { try { await prisma.$disconnect(); } catch {} } process.exit(1); });