import path from "node:path"; import fs from "fs-extra"; import * as cheerio from "cheerio"; import { chromium, type Browser } from "playwright"; import { nanoid } from "nanoid"; import { store } from "./db.js"; import { config } from "./config.js"; import { artifactAssetsDir, artifactIndexPath, safeAssetName, screenshotPath, screenshotsDir } from "./artifacts.js"; import { slugify } from "./slug.js"; import { writeConversationArtifact, writeFailureArtifact } from "./render.js"; import type { CaptureRow, NormalizedConversation, NormalizedMessage, ScreenshotVariant } from "./types.js"; interface ExtractedConversation { title: string; messages: NormalizedMessage[]; } function publicErrorFor(error: unknown): string { const message = error instanceof Error ? error.message : String(error); if (/captcha|verify|robot/i.test(message)) { return "ChatGPT asked for verification, so this public share could not be captured automatically."; } if (/timeout/i.test(message)) { return "The public ChatGPT share took too long to load."; } if (/unsupported|message/i.test(message)) { return "This public ChatGPT share could not be converted because its page format is not supported yet."; } return "This public ChatGPT share could not be captured."; } function timeoutSignal(ms: number): AbortSignal { const controller = new AbortController(); setTimeout(() => controller.abort(), ms).unref(); return controller.signal; } async function downloadAsset(assetUrl: string, slug: string): Promise { const response = await fetch(assetUrl, { signal: timeoutSignal(20_000), headers: { "user-agent": "aishare/0.1" } }); if (!response.ok) return null; const contentType = response.headers.get("content-type"); if (!contentType?.startsWith("image/")) return null; const bytes = Buffer.from(await response.arrayBuffer()); if (bytes.byteLength > 20 * 1024 * 1024) return null; const filename = safeAssetName(assetUrl, contentType, bytes); await fs.ensureDir(artifactAssetsDir(slug)); await fs.writeFile(path.join(artifactAssetsDir(slug), filename), bytes); return `/aishare/${slug}/assets/${filename}`; } async function localizeAssets(conversation: NormalizedConversation, slug: string): Promise<{ failures: string[] }> { const failures: string[] = []; for (const message of conversation.messages) { const $ = cheerio.load(message.html, {}, false); const images = $("img").toArray(); for (const image of images) { const current = $(image).attr("src"); if (!current) continue; let absolute: string; try { absolute = new URL(current, conversation.sourceUrl).toString(); } catch { $(image).removeAttr("src"); failures.push(current); continue; } if (!absolute.startsWith("https://")) { $(image).removeAttr("src"); failures.push(absolute); continue; } try { const local = await downloadAsset(absolute, slug); if (local) { $(image).attr("src", local); } else { $(image).removeAttr("src"); failures.push(absolute); } } catch { $(image).removeAttr("src"); failures.push(absolute); } } message.html = $.html(); } return { failures }; } async function extractConversation(browser: Browser, sourceUrl: string): Promise { const context = await browser.newContext({ viewport: { width: 1360, height: 900 }, deviceScaleFactor: 1, locale: "en-US" }); const page = await context.newPage(); page.setDefaultTimeout(config.captureTimeoutMs); await page.goto(sourceUrl, { waitUntil: "domcontentloaded", timeout: config.captureTimeoutMs }); await page.waitForLoadState("networkidle", { timeout: Math.min(config.captureTimeoutMs, 60_000) }).catch(() => undefined); await page.waitForTimeout(2500); const extracted = await page.evaluate(() => { function cleanHtml(root: Element): string { const clone = root.cloneNode(true) as Element; clone.querySelectorAll("script, style, iframe, object, embed, form, button, textarea, input, nav").forEach((node) => node.remove()); clone.querySelectorAll("*").forEach((element) => { Array.from(element.attributes).forEach((attr) => { const name = attr.name.toLowerCase(); const keep = ["href", "src", "alt", "title", "colspan", "rowspan"].includes(name); if (name.startsWith("on") || !keep) element.removeAttribute(attr.name); if ((name === "href" || name === "src") && /^javascript:/i.test(attr.value)) element.removeAttribute(attr.name); }); }); return clone.innerHTML.trim(); } function textToHtml(text: string): string { const escaped = text .replace(/&/g, "&") .replace(//g, ">"); return escaped .split(/\n{2,}/) .map((part) => `

${part.replace(/\n/g, "
")}

`) .join(""); } const title = document.querySelector("h1")?.textContent?.trim() || document.title.replace(/\s*[|-]\s*ChatGPT\s*$/i, "").trim() || "ChatGPT Share"; const roleNodes = Array.from(document.querySelectorAll("[data-message-author-role]")); const articleNodes = roleNodes.length ? [] : Array.from(document.querySelectorAll("main article")); const nodes = roleNodes.length ? roleNodes : articleNodes; const messages = nodes .map((node, index) => { const roleAttr = node.getAttribute("data-message-author-role") || ""; const role = roleAttr === "user" || roleAttr === "assistant" || roleAttr === "system" ? roleAttr : index % 2 === 0 ? "user" : "assistant"; const contentRoot = node.querySelector(".markdown") || node.querySelector("[data-message-content]") || node; const text = (contentRoot.textContent || "").replace(/\s+\n/g, "\n").trim(); const html = cleanHtml(contentRoot) || textToHtml(text); return { id: `m-${index + 1}`, role, text, html }; }) .filter((message) => message.text.length > 0); const bodyText = document.body.textContent || ""; if (/verify you are human|captcha|unusual activity/i.test(bodyText)) { throw new Error("ChatGPT verification or CAPTCHA page was shown."); } return { title, messages }; }); await context.close(); if (!extracted.messages.length) { throw new Error("Unsupported ChatGPT share page: no conversation messages were found."); } return extracted as ExtractedConversation; } async function generateScreenshotVariant(slug: string, variant: ScreenshotVariant): Promise { const browser = await chromium.launch({ headless: true }); try { const isMobile = variant.device === "mobile"; const context = await browser.newContext({ viewport: isMobile ? { width: 430, height: 932 } : { width: 1360, height: 900 }, deviceScaleFactor: 2, colorScheme: variant.theme, isMobile }); const page = await context.newPage(); const fileUrl = `file://${artifactIndexPath(slug)}?theme=${variant.theme}`; await page.goto(fileUrl, { waitUntil: "load", timeout: 60_000 }); const height = await page.evaluate(() => Math.ceil(document.documentElement.scrollHeight)); if (height > config.maxScreenshotHeight) { throw new Error(`Screenshot height ${height}px exceeds limit ${config.maxScreenshotHeight}px.`); } await fs.ensureDir(screenshotsDir(slug)); await page.screenshot({ path: screenshotPath(slug, `${variant.device}-${variant.theme}`), fullPage: true, type: "png" }); await context.close(); } finally { await browser.close(); } } async function generateScreenshots(slug: string): Promise<{ generated: string[]; failed: string[] }> { const variants: ScreenshotVariant[] = [ { device: "desktop", theme: "light" }, { device: "desktop", theme: "dark" }, { device: "mobile", theme: "light" }, { device: "mobile", theme: "dark" } ]; const generated: string[] = []; const failed: string[] = []; for (const variant of variants) { const name = `${variant.device}-${variant.theme}`; try { await generateScreenshotVariant(slug, variant); generated.push(name); } catch (error) { failed.push(`${name}: ${error instanceof Error ? error.message : String(error)}`); } } return { generated, failed }; } export async function runCapture(row: CaptureRow): Promise { let browser: Browser | null = null; const fallbackSlug = store.uniqueSlug(slugify(row.requested_slug || `chatgpt-share-${row.id.slice(0, 8)}`)); try { store.updateStatus(row.id, "capturing"); browser = await chromium.launch({ headless: true }); const extracted = await extractConversation(browser, row.source_url); await browser.close(); browser = null; const slug = store.uniqueSlug(slugify(row.requested_slug || extracted.title || fallbackSlug)); const conversation: NormalizedConversation = { provider: "chatgpt", title: extracted.title || "ChatGPT Share", sourceUrl: row.source_url, capturedAt: new Date().toISOString(), messages: extracted.messages }; store.updateStatus(row.id, "rendering"); const assetResult = await localizeAssets(conversation, slug); await writeConversationArtifact(slug, conversation); const screenshots = await generateScreenshots(slug); store.completeCapture(row.id, { slug, title: conversation.title, dataJson: JSON.stringify(conversation), metadataJson: JSON.stringify({ assetFailures: assetResult.failures, screenshots }) }); } catch (error) { if (browser) await browser.close().catch(() => undefined); const current = store.getCapture(row.id) || row; const slug = current.slug || fallbackSlug; store.failCapture(row.id, { slug, title: current.title || "Capture failed", errorPublic: publicErrorFor(error), errorDetail: error instanceof Error ? `${error.message}\n${error.stack || ""}` : String(error), metadataJson: JSON.stringify({ failedAt: new Date().toISOString() }) }); const failedRow = store.getCapture(row.id); if (failedRow) await writeFailureArtifact(failedRow); } } export function newCaptureId(): string { return nanoid(12); }