import { PrismaClient } from "@prisma/client";
import { PrismaMariaDb } from "@prisma/adapter-mariadb";
import "dotenv/config";
const rawUrl = process.env.DATABASE_URL;
if (!rawUrl) { console.error("DATABASE_URL not set"); process.exit(1); }
const base = rawUrl.replace("mysql://", "mariadb://");
const sep = base.includes("?") ? "&" : "?";
const connectionString = `${base}${sep}connection_limit=10&pool_timeout=15`;
const adapter = new PrismaMariaDb(connectionString);
const prisma = new PrismaClient({ adapter });
const USER_AGENT = "Mozilla/5.0 (compatible; EnrichBot/1.0; +https://www.zhuig.com)";
const FETCH_TIMEOUT_MS = 10000;
const RATE_LIMIT_MS = 2000;
async function delay(ms) {
return new Promise((r) => setTimeout(r, ms));
}
function resolveUrl(href, base) {
try {
return new URL(href, base).href;
} catch {
return null;
}
}
async function safeFetch(url) {
try {
return await fetch(url, {
method: "GET",
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
redirect: "follow",
headers: { "User-Agent": USER_AGENT },
});
} catch {
return null;
}
}
async function fetchOgData(websiteUrl) {
const result = { logo: null, description: null };
let url;
try {
url = new URL(websiteUrl);
if (!["http:", "https:"].includes(url.protocol)) return result;
} catch {
return result;
}
const res = await safeFetch(url.href);
if (!res || !res.ok) return result;
let html;
try {
html = await res.text();
} catch {
return result;
}
const ogImageMatch = html.match(/").replace(/"/g, '"').replace(/'/g, "'");
if (desc.trim().length >= 10) {
result.description = desc.trim();
}
}
if (!result.logo) {
const faviconMatch = html.match(/]*rel=["'](?:shortcut\s+)?icon["'][^>]*href=["']([^"']+)["']/i)
|| html.match(/]*href=["']([^"']+)["'][^>]*rel=["'](?:shortcut\s+)?icon["']/i);
if (faviconMatch) {
const favicon = resolveUrl(faviconMatch[1], url.href);
if (favicon) {
result.logo = favicon;
}
}
}
if (!result.logo) {
const domain = url.hostname;
const faviconGuesses = [`${url.protocol}//${domain}/favicon.ico`, `${url.protocol}//${domain}/apple-touch-icon.png`];
for (const guess of faviconGuesses) {
const headRes = await safeFetch(guess);
if (headRes && headRes.ok) {
const ct = headRes.headers.get("content-type") || "";
if (ct.startsWith("image/") || ct.includes("icon") || guess.endsWith(".ico")) {
result.logo = guess;
break;
}
}
}
}
return result;
}
async function tryClearbitLogo(domain) {
try {
const url = `https://logo.clearbit.com/${encodeURIComponent(domain)}`;
const res = await fetch(url, {
method: "HEAD",
signal: AbortSignal.timeout(8000),
headers: { "User-Agent": USER_AGENT },
});
if (res.ok) {
const ct = res.headers.get("content-type") || "";
if (ct.startsWith("image/")) return url;
}
} catch {
// Clearbit unavailable, expected
}
return null;
}
async function enrichTool(tool) {
let domain;
try {
domain = new URL(tool.websiteUrl).hostname;
} catch {
return null;
}
const needsLogo = !tool.logoUrl;
const needsDesc = !tool.description || tool.description.length < 100;
if (!needsLogo && !needsDesc) return null;
const updates = {};
const findings = [];
const og = await fetchOgData(tool.websiteUrl);
if (needsLogo) {
if (og.logo) {
updates.logoUrl = og.logo;
findings.push("logo");
} else {
const clearbit = await tryClearbitLogo(domain);
if (clearbit) {
updates.logoUrl = clearbit;
findings.push("logo(clearbit)");
}
}
}
if (needsDesc && og.description && og.description.length > (tool.description?.length || 0)) {
updates.description = og.description.slice(0, 500);
findings.push("desc");
}
if (Object.keys(updates).length > 0) {
await prisma.tool.update({ where: { id: tool.id }, data: updates });
}
return findings;
}
async function main() {
const startTime = new Date();
const tools = await prisma.tool.findMany({
where: { status: "published", websiteUrl: { not: "" } },
select: {
id: true,
name: true,
slug: true,
websiteUrl: true,
logoUrl: true,
description: true,
},
});
const needingEnrichment = tools.filter((t) => {
const needsLogo = !t.logoUrl;
const needsDesc = !t.description || t.description.length < 100;
return needsLogo || needsDesc;
});
console.log(`Found ${tools.length} published tools, ${needingEnrichment.length} need enrichment\n`);
let enriched = 0;
let logoFound = 0;
let descFound = 0;
let failed = 0;
const errors = [];
for (let i = 0; i < needingEnrichment.length; i++) {
const tool = needingEnrichment[i];
const idx = i + 1;
const total = needingEnrichment.length;
process.stdout.write(`[${idx}/${total}] ${tool.name}: `);
try {
const findings = await enrichTool(tool);
if (findings && findings.length > 0) {
enriched++;
if (findings.includes("logo") || findings.includes("logo(clearbit)")) logoFound++;
if (findings.includes("desc")) descFound++;
process.stdout.write(`${findings.join(", ")} ✓\n`);
} else {
process.stdout.write(`nothing to enrich\n`);
}
} catch (err) {
failed++;
errors.push(`${tool.name}: ${err.message}`);
process.stdout.write(`error: ${err.message}\n`);
}
if (idx < total) {
await delay(RATE_LIMIT_MS);
}
}
const endTime = new Date();
const duration = endTime.getTime() - startTime.getTime();
console.log(`\nDone.`);
console.log(` Tools processed: ${needingEnrichment.length}`);
console.log(` Enriched: ${enriched} (logo: ${logoFound}, description: ${descFound})`);
console.log(` Failed: ${failed}`);
console.log(` Duration: ${(duration / 1000).toFixed(1)}s`);
if (errors.length > 0) {
console.log(`\nErrors:`);
errors.forEach((e) => console.log(` - ${e}`));
}
await prisma.$disconnect();
}
main().catch(async (err) => {
console.error("Fatal:", err.message);
if (prisma) {
try { await prisma.$disconnect(); } catch {}
}
process.exit(1);
});