import { prisma } from "@/lib/prisma"; interface LogoFetchConfig { enabledMethods: string[]; commonPaths: string[]; faviconServices: { name: string; url: string; enabled: boolean }[]; minFileSize: number; requestTimeout: number; } const logoCache = new Map(); const CACHE_TTL = 3600000; function getCachedLogo(domain: string): string | null { const cached = logoCache.get(domain); if (cached && Date.now() - cached.timestamp < CACHE_TTL) { return cached.url; } if (cached) logoCache.delete(domain); return null; } function setCachedLogo(domain: string, url: string) { if (logoCache.size > 500) { const keys = Array.from(logoCache.keys()); for (let i = 0; i < 100 && i < keys.length; i++) { logoCache.delete(keys[i]); } } logoCache.set(domain, { url, timestamp: Date.now() }); } const DEFAULT_CONFIG: LogoFetchConfig = { enabledMethods: [ "html_link_icon", "html_meta_og", "html_manifest", "html_img_logo", "html_header_logo", "html_svg_logo", "common_paths", "favicon_services", "brandfetch", "screenshot_fallback", ], commonPaths: [ "/apple-touch-icon.png", "/apple-touch-icon-precomposed.png", "/apple-touch-icon-180x180.png", "/apple-touch-icon-152x152.png", "/logo.png", "/logo.svg", "/images/logo.png", "/images/logo.svg", "/img/logo.png", "/img/logo.svg", "/assets/logo.png", "/assets/logo.svg", "/assets/images/logo.png", "/assets/img/logo.png", "/static/logo.png", "/static/logo.svg", "/static/images/logo.png", "/public/logo.png", "/favicon-192x192.png", "/favicon-32x32.png", "/android-chrome-192x192.png", "/mstile-150x150.png", "/icons/icon-192x192.png", "/icons/icon-512x512.png", "/icons/apple-touch-icon.png", "/_next/static/media/logo.svg", "/_next/static/media/logo.png", "/favicon.ico", ], faviconServices: [ { name: "favicon.im", url: "https://favicon.im/{domain}", enabled: true }, { name: "Google Favicons 128", url: "https://www.google.com/s2/favicons?domain={domain}&sz=128", enabled: true }, { name: "Google Favicons 64", url: "https://www.google.com/s2/favicons?domain={domain}&sz=64", enabled: true }, { name: "Clearbit", url: "https://logo.clearbit.com/{domain}", enabled: true }, { name: "DuckDuckGo", url: "https://icons.duckduckgo.com/ip3/{domain}.ico", enabled: true }, { name: "Icon Horse", url: "https://icon.horse/icon/{domain}", enabled: true }, { name: "Yandex Favicon", url: "https://favicon.yandex.net/favicon/v2/{domain}?size=128", enabled: true }, { name: "Splitbee", url: "https://icon.splitbee.io/{domain}", enabled: true }, { name: "The Favicon API", url: "https://favicone.com/{domain}?s=128", enabled: true }, { name: "Favicon Grabr", url: "https://favicongrabr.com/api/grab/{domain}", enabled: false }, { name: "Besticon", url: "https://besticon-demo.herokuapp.com/icon?url={domain}&size=128", enabled: false }, ], minFileSize: 50, requestTimeout: 10000, }; async function getConfig(): Promise { try { const row = await prisma.systemConfig.findUnique({ where: { key: "logoFetchConfig" } }); if (row?.value && typeof row.value === "object") { return { ...DEFAULT_CONFIG, ...(row.value as Partial) }; } } catch (err) { console.warn("logo-fetcher: Failed to load logoFetchConfig from database, using defaults", err); } return DEFAULT_CONFIG; } const UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"; function resolveUrl(href: string, origin: string): string { if (href.startsWith("//")) return "https:" + href; if (href.startsWith("/")) return `${origin}${href}`; if (href.startsWith("http")) return href; return `${origin}/${href}`; } async function checkImageUrl(url: string, timeout: number, minSize: number): Promise { try { const res = await fetch(url, { method: "GET", signal: AbortSignal.timeout(timeout), headers: { "User-Agent": UA, Accept: "image/*,*/*", }, redirect: "follow", }); if (res.ok) { const ct = res.headers.get("content-type") || ""; if (ct.includes("image") || ct.includes("icon") || ct.includes("svg") || ct.includes("octet-stream")) { const buf = await res.arrayBuffer(); if (buf.byteLength > minSize) { return res.url || url; } } } } catch (err) { console.warn("logo-fetcher: Failed to check image URL", err); } return ""; } async function fetchManifestIcons(origin: string, html: string, timeout: number, minSize: number): Promise { const manifestPatterns = [ /]+rel=["']manifest["'][^>]+href=["']([^"']+)["']/i, /]+href=["']([^"']+)["'][^>]+rel=["']manifest["']/i, ]; let manifestHref = ""; for (const pattern of manifestPatterns) { const match = html.match(pattern); if (match?.[1]) { manifestHref = resolveUrl(match[1], origin); break; } } if (!manifestHref) { for (const candidate of ["/manifest.json", "/manifest.webmanifest", "/site.webmanifest"]) { try { const res = await fetch(`${origin}${candidate}`, { method: "HEAD", signal: AbortSignal.timeout(timeout), headers: { "User-Agent": UA }, }); if (res.ok) { manifestHref = `${origin}${candidate}`; break; } } catch (err) { console.warn("logo-fetcher: Failed to probe manifest candidate path", err); } } } if (!manifestHref) return ""; try { const res = await fetch(manifestHref, { signal: AbortSignal.timeout(timeout), headers: { "User-Agent": UA, Accept: "application/json,*/*" }, }); if (res.ok) { const data = await res.json(); const icons: { src: string; sizes?: string; type?: string }[] = data.icons || []; const sorted = icons.sort((a, b) => { const sizeA = parseInt((a.sizes || "0").split("x")[0]) || 0; const sizeB = parseInt((b.sizes || "0").split("x")[0]) || 0; return sizeB - sizeA; }); for (const icon of sorted) { if (!icon.src) continue; const iconUrl = resolveUrl(icon.src, origin); const valid = await checkImageUrl(iconUrl, timeout, minSize); if (valid) return valid; } } } catch (err) { console.warn("logo-fetcher: Failed to fetch manifest icons", err); } return ""; } function extractSvgAsDataUri(html: string): string { const svgPatterns = [ /]+class=["'][^"']*(?:logo|brand|site-icon)[^"']*["'][^>]*>[\s\S]*?<\/svg>/gi, /]+id=["'][^"']*(?:logo|brand)[^"']*["'][^>]*>[\s\S]*?<\/svg>/gi, /]+aria-label=["'][^"']*(?:logo|Logo)[^"']*["'][^>]*>[\s\S]*?<\/svg>/gi, ]; for (const pattern of svgPatterns) { const match = html.match(pattern); if (match?.[0]) { const svg = match[0].trim(); if (svg.length > 50 && svg.length < 50000) { const encoded = svg .replace(/"/g, "'") .replace(/%/g, "%25") .replace(/#/g, "%23") .replace(/{/g, "%7B") .replace(/}/g, "%7D") .replace(//g, "%3E"); return `data:image/svg+xml,${encoded}`; } } } return ""; } async function fetchSvgLogo(origin: string, html: string, timeout: number, minSize: number): Promise { const dataUri = extractSvgAsDataUri(html); if (dataUri) return dataUri; const svgSrcPatterns = [ /]+(?:src|data-src)=["']([^"']*(?:logo|brand)[^"']*\.svg)["']/gi, /]+(?:src|data-src)=["']([^"']*\.svg)["'][^>]+class=["'][^"']*(?:logo|brand)[^"']*["']/gi, /]+data=["']([^"']*\.svg)["']/gi, /]+src=["']([^"']*\.svg)["']/gi, ]; const svgCandidates: string[] = []; for (const pattern of svgSrcPatterns) { let match: RegExpExecArray | null; while ((match = pattern.exec(html)) !== null) { if (match[1] && !match[1].includes("data:")) { svgCandidates.push(resolveUrl(match[1], origin)); } } } for (const c of svgCandidates) { try { const res = await fetch(c, { signal: AbortSignal.timeout(timeout), headers: { "User-Agent": UA, Accept: "image/svg+xml,*/*" }, redirect: "follow", }); if (res.ok) { const ct = res.headers.get("content-type") || ""; if (ct.includes("svg") || ct.includes("image")) { const text = await res.text(); if (text.includes(" 50) { return res.url || c; } } } } catch (err) { console.warn("logo-fetcher: Failed to fetch SVG logo candidate", err); } } return ""; } async function fetchHeaderLogo(origin: string, html: string, timeout: number, minSize: number): Promise { const headerPatterns = [ /]*>[\s\S]*?]+(?:src|data-src)=["']([^"']+)["'][^>]*>[\s\S]*?<\/header>/gi, /]*>[\s\S]*?]+(?:src|data-src)=["']([^"']+)["'][^>]*>[\s\S]*?<\/nav>/gi, /]+class=["'][^"']*(?:header|navbar|topbar|nav-bar|site-header|main-header)[^"']*["'][^>]*>[\s\S]*?]+(?:src|data-src)=["']([^"']+)["']/gi, ]; for (const pattern of headerPatterns) { let match: RegExpExecArray | null; while ((match = pattern.exec(html)) !== null) { const src = match[1] || match[2]; if (src && !src.includes("data:") && src.length > 5) { const resolved = resolveUrl(src, origin); if (resolved.match(/\.(png|jpg|jpeg|svg|webp|ico)(\?|$)/i)) { const valid = await checkImageUrl(resolved, timeout, minSize); if (valid) return valid; } } } } const headerSection = html.match(/]*>([\s\S]*?)<\/header>/i); if (headerSection) { const imgInHeader = /]+(?:src|data-src)=["']([^"']+)["']/gi; let imgMatch: RegExpExecArray | null; while ((imgMatch = imgInHeader.exec(headerSection[1])) !== null) { if (imgMatch[1] && !imgMatch[1].includes("data:") && imgMatch[1].length > 5) { const resolved = resolveUrl(imgMatch[1], origin); const valid = await checkImageUrl(resolved, timeout, minSize); if (valid) return valid; } } } return ""; } async function fetchBrandfetch(domain: string, timeout: number): Promise { try { const res = await fetch(`https://api.brandfetch.io/v2/brands/${domain}`, { signal: AbortSignal.timeout(timeout), headers: { "User-Agent": UA, Accept: "application/json", }, }); if (res.ok) { const data = await res.json(); const logos: { formats: { src: string; format: string; size?: number; background?: string }[] }[] = data.logos || []; if (logos.length > 0 && logos[0].formats?.length > 0) { const sorted = logos[0].formats.sort((a, b) => { const prioA = a.format === "svg" ? 100 : (a.size || 0); const prioB = b.format === "svg" ? 100 : (b.size || 0); return prioB - prioA; }); const best = sorted[0]; if (best?.src) return best.src; } const icons: { formats: { src: string; format: string; size?: number }[] }[] = data.icons || []; if (icons.length > 0 && icons[0].formats?.length > 0) { const sorted = icons[0].formats.sort((a, b) => (b.size || 0) - (a.size || 0)); const best = sorted[0]; if (best?.src) return best.src; } } } catch (err) { console.warn("logo-fetcher: Failed to fetch brand from Brandfetch", err); } return ""; } async function fetchScreenshotFallback(domain: string, timeout: number): Promise { const screenshotServices = [ `https://image.thum.io/get/width/128/crop/128/${domain}`, `https://api.microlink.io/?url=https://${domain}&screenshot=true&meta=false&embed=screenshot.url`, ]; for (const serviceUrl of screenshotServices) { try { const res = await fetch(serviceUrl, { signal: AbortSignal.timeout(timeout * 2), headers: { "User-Agent": UA }, redirect: "follow", }); if (res.ok) { const ct = res.headers.get("content-type") || ""; if (ct.includes("json")) { const data = await res.json(); const imageUrl = data?.screenshot?.url || data?.data?.screenshot?.url; if (imageUrl && typeof imageUrl === "string") { return imageUrl; } } else if (ct.includes("image")) { const buf = await res.arrayBuffer(); if (buf.byteLength > 500) { return res.url || serviceUrl; } } } } catch (err) { console.warn("logo-fetcher: Failed to fetch screenshot fallback", err); } } return ""; } export interface LogoFetchResult { url: string; method: string; } export async function fetchLogoWithMethod(websiteUrl: string, html?: string): Promise { try { const urlObj = new URL(websiteUrl); const domain = urlObj.hostname; if (!html) { const cached = getCachedLogo(domain); if (cached !== null) return { url: cached, method: cached ? "cache" : "" }; } const cfg = await getConfig(); const origin = urlObj.origin; const timeout = cfg.requestTimeout || 10000; const minSize = cfg.minFileSize || 50; if (html && cfg.enabledMethods.includes("html_link_icon")) { const linkPatterns = [ /]+rel=["'](?:apple-touch-icon(?:-precomposed)?|icon|shortcut icon|mask-icon)["'][^>]+(?:href|content)=["']([^"']+)["']/gi, /]+(?:href|content)=["']([^"']+)["'][^>]+rel=["'](?:apple-touch-icon(?:-precomposed)?|icon|shortcut icon|mask-icon)["']/gi, /]+rel=["'](?:apple-touch-icon(?:-precomposed)?|icon|shortcut icon|mask-icon)["'][^>]+(?:href|content)=([^>\s]+)/gi, /]+(?:href|content)=([^>\s]+)[^>]+rel=["'](?:apple-touch-icon(?:-precomposed)?|icon|shortcut icon|mask-icon)["']/gi, /]+sizes=["'][^"']*192[^"']*["'][^>]+href=["']([^"']+)["']/gi, /]+sizes=["'][^"']*180[^"']*["'][^>]+href=["']([^"']+)["']/gi, /]+sizes=["'][^"']*152[^"']*["'][^>]+href=["']([^"']+)["']/gi, ]; const candidates: string[] = []; for (const pattern of linkPatterns) { let match: RegExpExecArray | null; while ((match = pattern.exec(html)) !== null) { const href = match[1]; if (!href || href.includes("data:")) continue; const resolved = resolveUrl(href, origin); if (resolved.includes("apple-touch-icon") || resolved.includes("icon-") || resolved.includes("192") || resolved.includes("180")) { const valid = await checkImageUrl(resolved, timeout, minSize); if (valid) { setCachedLogo(domain, valid); return { url: valid, method: "HTML Link图标" }; } } candidates.push(resolved); } } for (const c of candidates) { if (!c.includes("favicon.im")) { const valid = await checkImageUrl(c, timeout, minSize); if (valid) { setCachedLogo(domain, valid); return { url: valid, method: "HTML Link图标" }; } } } } if (html && cfg.enabledMethods.includes("html_meta_og")) { const metaPatterns = [ /]+property=["']og:image["'][^>]+content=["']([^"']+)["']/i, /]+content=["']([^"']+)["'][^>]+property=["']og:image["']/i, /]+name=["']twitter:image["'][^>]+content=["']([^"']+)["']/i, /]+name=["']twitter:image:src["'][^>]+content=["']([^"']+)["']/i, /]+property=["']og:image["'][^>]+content=([^>\s]+)/i, /]+content=([^>\s]+)[^>]+property=["']og:image["']/i, ]; for (const pattern of metaPatterns) { const match = html.match(pattern); if (match?.[1] && !match[1].includes("data:")) { const resolved = resolveUrl(match[1], origin); const valid = await checkImageUrl(resolved, timeout, minSize); if (valid) { setCachedLogo(domain, valid); return { url: valid, method: "OG/Twitter图片" }; } } } } if (html && cfg.enabledMethods.includes("html_manifest")) { const manifestResult = await fetchManifestIcons(origin, html, timeout, minSize); if (manifestResult) { setCachedLogo(domain, manifestResult); return { url: manifestResult, method: "PWA Manifest" }; } } if (html && cfg.enabledMethods.includes("html_img_logo")) { const imgPatterns = [ /]+(?:src|data-src)=["']([^"']*(?:logo|brand)[^"']*)["']/gi, /]+(?:src|data-src)=["']([^"']*(?:Logo|Brand)[^"']*)["']/gi, /]+class=["'][^"']*(?:logo|brand|site-icon)[^"']*["'][^>]+(?:src|data-src)=["']([^"']+)["']/gi, /]+(?:src|data-src)=["']([^"']+)["'][^>]+class=["'][^"']*(?:logo|brand|site-icon)[^"']*["']/gi, /]+alt=["'][^"']*(?:logo|Logo)[^"']*["'][^>]+(?:src|data-src)=["']([^"']+)["']/gi, ]; const imgCandidates: string[] = []; for (const pattern of imgPatterns) { let imgMatch: RegExpExecArray | null; while ((imgMatch = pattern.exec(html)) !== null) { if (imgMatch[1] && !imgMatch[1].includes("data:") && imgMatch[1].length > 5) { imgCandidates.push(resolveUrl(imgMatch[1], origin)); } } } for (const c of imgCandidates) { const valid = await checkImageUrl(c, timeout, minSize); if (valid) { setCachedLogo(domain, valid); return { url: valid, method: "HTML Logo图片" }; } } } if (html && cfg.enabledMethods.includes("html_header_logo")) { const headerResult = await fetchHeaderLogo(origin, html, timeout, minSize); if (headerResult) { setCachedLogo(domain, headerResult); return { url: headerResult, method: "Header区域Logo" }; } } if (html && cfg.enabledMethods.includes("html_svg_logo")) { const svgResult = await fetchSvgLogo(origin, html, timeout, minSize); if (svgResult) { setCachedLogo(domain, svgResult); return { url: svgResult, method: "SVG Logo" }; } } if (cfg.enabledMethods.includes("common_paths")) { for (const p of cfg.commonPaths) { const testUrl = `${origin}${p}`; const valid = await checkImageUrl(testUrl, timeout, minSize); if (valid) { setCachedLogo(domain, valid); return { url: valid, method: "常见路径探测" }; } } } if (cfg.enabledMethods.includes("favicon_services")) { for (const svc of cfg.faviconServices) { if (!svc.enabled) continue; try { const serviceUrl = svc.url.replace("{domain}", domain); if (svc.url.includes("favicongrabr") || svc.url.includes("icon.horse")) { const apiRes = await fetch(serviceUrl, { signal: AbortSignal.timeout(timeout), headers: { "User-Agent": UA }, }); if (apiRes.ok) { const ct = apiRes.headers.get("content-type") || ""; if (ct.includes("json")) { const data = await apiRes.json(); const icons: { src: string; sizes?: string }[] = data.icons || []; const sorted = icons.sort((a, b) => { const sizeA = parseInt((a.sizes || "0").split("x")[0]) || 0; const sizeB = parseInt((b.sizes || "0").split("x")[0]) || 0; return sizeB - sizeA; }); const best = sorted.find((i) => i.src.includes("apple-touch-icon")) || sorted.find((i) => i.sizes === "128x128" || i.sizes === "192x192") || sorted[0]; if (best?.src) { const valid = await checkImageUrl(best.src, timeout, minSize); if (valid) { setCachedLogo(domain, valid); return { url: valid, method: `Favicon服务(${svc.name})` }; } } } else if (ct.includes("image")) { const buf = await apiRes.arrayBuffer(); if (buf.byteLength > minSize) { const result = apiRes.url || serviceUrl; setCachedLogo(domain, result); return { url: result, method: `Favicon服务(${svc.name})` }; } } } } else { const valid = await checkImageUrl(serviceUrl, timeout, minSize); if (valid) { setCachedLogo(domain, valid); return { url: valid, method: `Favicon服务(${svc.name})` }; } } } catch (err) { console.warn("logo-fetcher: Failed to fetch favicon from service", err); } } } if (cfg.enabledMethods.includes("brandfetch")) { const brandfetchResult = await fetchBrandfetch(domain, timeout); if (brandfetchResult) { setCachedLogo(domain, brandfetchResult); return { url: brandfetchResult, method: "Brandfetch品牌库" }; } } if (cfg.enabledMethods.includes("screenshot_fallback")) { const screenshotResult = await fetchScreenshotFallback(domain, timeout); if (screenshotResult) { setCachedLogo(domain, screenshotResult); return { url: screenshotResult, method: "截图兜底" }; } } const fallbackUrl = `${origin}/favicon.ico`; const valid = await checkImageUrl(fallbackUrl, timeout, minSize); if (valid) { setCachedLogo(domain, valid); return { url: valid, method: "Favicon默认路径" }; } setCachedLogo(domain, ""); return { url: "", method: "" }; } catch (err) { console.warn("logo-fetcher: Failed to fetch logo for URL", err); return { url: "", method: "" }; } } export async function fetchLogo(websiteUrl: string, html?: string): Promise { const result = await fetchLogoWithMethod(websiteUrl, html); return result.url; }