feat: 首次推送到Gitea - 完整项目代码 + 安全加固 + 知识库
This commit is contained in:
ZhuiGuangAI Dev
@@ -0,0 +1,247 @@
|
||||
import { PrismaClient } from "@prisma/client";
|
||||
import { PrismaMariaDb } from "@prisma/adapter-mariadb";
|
||||
import "dotenv/config";
|
||||
|
||||
const rawUrl = process.env.DATABASE_URL;
|
||||
if (!rawUrl) { console.error("DATABASE_URL not set"); process.exit(1); }
|
||||
const base = rawUrl.replace("mysql://", "mariadb://");
|
||||
const sep = base.includes("?") ? "&" : "?";
|
||||
const connectionString = `${base}${sep}connection_limit=10&pool_timeout=15`;
|
||||
const adapter = new PrismaMariaDb(connectionString);
|
||||
const prisma = new PrismaClient({ adapter });
|
||||
|
||||
const USER_AGENT = "Mozilla/5.0 (compatible; EnrichBot/1.0; +https://www.zhuig.com)";
|
||||
const FETCH_TIMEOUT_MS = 10000;
|
||||
const RATE_LIMIT_MS = 2000;
|
||||
|
||||
async function delay(ms) {
|
||||
return new Promise((r) => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
function resolveUrl(href, base) {
|
||||
try {
|
||||
return new URL(href, base).href;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function safeFetch(url) {
|
||||
try {
|
||||
return await fetch(url, {
|
||||
method: "GET",
|
||||
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
|
||||
redirect: "follow",
|
||||
headers: { "User-Agent": USER_AGENT },
|
||||
});
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchOgData(websiteUrl) {
|
||||
const result = { logo: null, description: null };
|
||||
|
||||
let url;
|
||||
try {
|
||||
url = new URL(websiteUrl);
|
||||
if (!["http:", "https:"].includes(url.protocol)) return result;
|
||||
} catch {
|
||||
return result;
|
||||
}
|
||||
|
||||
const res = await safeFetch(url.href);
|
||||
if (!res || !res.ok) return result;
|
||||
|
||||
let html;
|
||||
try {
|
||||
html = await res.text();
|
||||
} catch {
|
||||
return result;
|
||||
}
|
||||
|
||||
const ogImageMatch = html.match(/<meta\s+property=["']og:image["']\s+content=["']([^"']+)["']/i);
|
||||
if (ogImageMatch) {
|
||||
result.logo = resolveUrl(ogImageMatch[1], url.href);
|
||||
}
|
||||
|
||||
const ogDescMatch = html.match(/<meta\s+property=["']og:description["']\s+content=["']([^"']+)["']/i);
|
||||
if (ogDescMatch) {
|
||||
const desc = ogDescMatch[1].replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'");
|
||||
if (desc.trim().length >= 10) {
|
||||
result.description = desc.trim();
|
||||
}
|
||||
}
|
||||
|
||||
if (!result.logo) {
|
||||
const faviconMatch = html.match(/<link\s+[^>]*rel=["'](?:shortcut\s+)?icon["'][^>]*href=["']([^"']+)["']/i)
|
||||
|| html.match(/<link\s+[^>]*href=["']([^"']+)["'][^>]*rel=["'](?:shortcut\s+)?icon["']/i);
|
||||
if (faviconMatch) {
|
||||
const favicon = resolveUrl(faviconMatch[1], url.href);
|
||||
if (favicon) {
|
||||
result.logo = favicon;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!result.logo) {
|
||||
const domain = url.hostname;
|
||||
const faviconGuesses = [`${url.protocol}//${domain}/favicon.ico`, `${url.protocol}//${domain}/apple-touch-icon.png`];
|
||||
for (const guess of faviconGuesses) {
|
||||
const headRes = await safeFetch(guess);
|
||||
if (headRes && headRes.ok) {
|
||||
const ct = headRes.headers.get("content-type") || "";
|
||||
if (ct.startsWith("image/") || ct.includes("icon") || guess.endsWith(".ico")) {
|
||||
result.logo = guess;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async function tryClearbitLogo(domain) {
|
||||
try {
|
||||
const url = `https://logo.clearbit.com/${encodeURIComponent(domain)}`;
|
||||
const res = await fetch(url, {
|
||||
method: "HEAD",
|
||||
signal: AbortSignal.timeout(8000),
|
||||
headers: { "User-Agent": USER_AGENT },
|
||||
});
|
||||
if (res.ok) {
|
||||
const ct = res.headers.get("content-type") || "";
|
||||
if (ct.startsWith("image/")) return url;
|
||||
}
|
||||
} catch {
|
||||
// Clearbit unavailable, expected
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function enrichTool(tool) {
|
||||
let domain;
|
||||
try {
|
||||
domain = new URL(tool.websiteUrl).hostname;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
const needsLogo = !tool.logoUrl;
|
||||
const needsDesc = !tool.description || tool.description.length < 100;
|
||||
|
||||
if (!needsLogo && !needsDesc) return null;
|
||||
|
||||
const updates = {};
|
||||
const findings = [];
|
||||
|
||||
const og = await fetchOgData(tool.websiteUrl);
|
||||
|
||||
if (needsLogo) {
|
||||
if (og.logo) {
|
||||
updates.logoUrl = og.logo;
|
||||
findings.push("logo");
|
||||
} else {
|
||||
const clearbit = await tryClearbitLogo(domain);
|
||||
if (clearbit) {
|
||||
updates.logoUrl = clearbit;
|
||||
findings.push("logo(clearbit)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (needsDesc && og.description && og.description.length > (tool.description?.length || 0)) {
|
||||
updates.description = og.description.slice(0, 500);
|
||||
findings.push("desc");
|
||||
}
|
||||
|
||||
if (Object.keys(updates).length > 0) {
|
||||
await prisma.tool.update({ where: { id: tool.id }, data: updates });
|
||||
}
|
||||
|
||||
return findings;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const startTime = new Date();
|
||||
|
||||
const tools = await prisma.tool.findMany({
|
||||
where: { status: "published", websiteUrl: { not: "" } },
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
slug: true,
|
||||
websiteUrl: true,
|
||||
logoUrl: true,
|
||||
description: true,
|
||||
},
|
||||
});
|
||||
|
||||
const needingEnrichment = tools.filter((t) => {
|
||||
const needsLogo = !t.logoUrl;
|
||||
const needsDesc = !t.description || t.description.length < 100;
|
||||
return needsLogo || needsDesc;
|
||||
});
|
||||
|
||||
console.log(`Found ${tools.length} published tools, ${needingEnrichment.length} need enrichment\n`);
|
||||
|
||||
let enriched = 0;
|
||||
let logoFound = 0;
|
||||
let descFound = 0;
|
||||
let failed = 0;
|
||||
const errors = [];
|
||||
|
||||
for (let i = 0; i < needingEnrichment.length; i++) {
|
||||
const tool = needingEnrichment[i];
|
||||
const idx = i + 1;
|
||||
const total = needingEnrichment.length;
|
||||
|
||||
process.stdout.write(`[${idx}/${total}] ${tool.name}: `);
|
||||
|
||||
try {
|
||||
const findings = await enrichTool(tool);
|
||||
|
||||
if (findings && findings.length > 0) {
|
||||
enriched++;
|
||||
if (findings.includes("logo") || findings.includes("logo(clearbit)")) logoFound++;
|
||||
if (findings.includes("desc")) descFound++;
|
||||
process.stdout.write(`${findings.join(", ")} ✓\n`);
|
||||
} else {
|
||||
process.stdout.write(`nothing to enrich\n`);
|
||||
}
|
||||
} catch (err) {
|
||||
failed++;
|
||||
errors.push(`${tool.name}: ${err.message}`);
|
||||
process.stdout.write(`error: ${err.message}\n`);
|
||||
}
|
||||
|
||||
if (idx < total) {
|
||||
await delay(RATE_LIMIT_MS);
|
||||
}
|
||||
}
|
||||
|
||||
const endTime = new Date();
|
||||
const duration = endTime.getTime() - startTime.getTime();
|
||||
|
||||
console.log(`\nDone.`);
|
||||
console.log(` Tools processed: ${needingEnrichment.length}`);
|
||||
console.log(` Enriched: ${enriched} (logo: ${logoFound}, description: ${descFound})`);
|
||||
console.log(` Failed: ${failed}`);
|
||||
console.log(` Duration: ${(duration / 1000).toFixed(1)}s`);
|
||||
|
||||
if (errors.length > 0) {
|
||||
console.log(`\nErrors:`);
|
||||
errors.forEach((e) => console.log(` - ${e}`));
|
||||
}
|
||||
|
||||
await prisma.$disconnect();
|
||||
}
|
||||
|
||||
main().catch(async (err) => {
|
||||
console.error("Fatal:", err.message);
|
||||
if (prisma) {
|
||||
try { await prisma.$disconnect(); } catch {}
|
||||
}
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user