aishare/src/capture.ts

import path from "node:path";
import fs from "fs-extra";
import * as cheerio from "cheerio";
import { chromium, type Browser } from "playwright";
import { nanoid } from "nanoid";
import { store } from "./db.js";
import { config } from "./config.js";
import { artifactAssetsDir, artifactIndexPath, safeAssetName, screenshotPath, screenshotsDir } from "./artifacts.js";
import { slugify } from "./slug.js";
import { writeConversationArtifact, writeFailureArtifact } from "./render.js";
import type { CaptureRow, NormalizedConversation, NormalizedMessage, ScreenshotVariant } from "./types.js";

interface ExtractedConversation {
  title: string;
  messages: NormalizedMessage[];
}

function publicErrorFor(error: unknown): string {
  const message = error instanceof Error ? error.message : String(error);
  if (/captcha|verify|robot/i.test(message)) {
    return "ChatGPT asked for verification, so this public share could not be captured automatically.";
  }
  if (/timeout/i.test(message)) {
    return "The public ChatGPT share took too long to load.";
  }
  if (/unsupported|message/i.test(message)) {
    return "This public ChatGPT share could not be converted because its page format is not supported yet.";
  }
  return "This public ChatGPT share could not be captured.";
}

function timeoutSignal(ms: number): AbortSignal {
  const controller = new AbortController();
  setTimeout(() => controller.abort(), ms).unref();
  return controller.signal;
}

async function downloadAsset(assetUrl: string, slug: string): Promise<string | null> {
  const response = await fetch(assetUrl, {
    signal: timeoutSignal(20_000),
    headers: {
      "user-agent": "aishare/0.1"
    }
  });
  if (!response.ok) return null;
  const contentType = response.headers.get("content-type");
  if (!contentType?.startsWith("image/")) return null;
  const bytes = Buffer.from(await response.arrayBuffer());
  if (bytes.byteLength > 20 * 1024 * 1024) return null;
  const filename = safeAssetName(assetUrl, contentType, bytes);
  await fs.ensureDir(artifactAssetsDir(slug));
  await fs.writeFile(path.join(artifactAssetsDir(slug), filename), bytes);
  return `/aishare/${slug}/assets/${filename}`;
}

async function localizeAssets(conversation: NormalizedConversation, slug: string): Promise<{ failures: string[] }> {
  const failures: string[] = [];
  for (const message of conversation.messages) {
    const $ = cheerio.load(message.html, {}, false);
    const images = $("img").toArray();
    for (const image of images) {
      const current = $(image).attr("src");
      if (!current) continue;
      let absolute: string;
      try {
        absolute = new URL(current, conversation.sourceUrl).toString();
      } catch {
        $(image).removeAttr("src");
        failures.push(current);
        continue;
      }
      if (!absolute.startsWith("https://")) {
        $(image).removeAttr("src");
        failures.push(absolute);
        continue;
      }
      try {
        const local = await downloadAsset(absolute, slug);
        if (local) {
          $(image).attr("src", local);
        } else {
          $(image).removeAttr("src");
          failures.push(absolute);
        }
      } catch {
        $(image).removeAttr("src");
        failures.push(absolute);
      }
    }
    message.html = $.html();
  }
  return { failures };
}

async function extractConversation(browser: Browser, sourceUrl: string): Promise<ExtractedConversation> {
  const context = await browser.newContext({
    viewport: { width: 1360, height: 900 },
    deviceScaleFactor: 1,
    locale: "en-US"
  });
  const page = await context.newPage();
  page.setDefaultTimeout(config.captureTimeoutMs);
  await page.goto(sourceUrl, { waitUntil: "domcontentloaded", timeout: config.captureTimeoutMs });
  await page.waitForLoadState("networkidle", { timeout: Math.min(config.captureTimeoutMs, 60_000) }).catch(() => undefined);
  await page.waitForTimeout(2500);

  const extracted = await page.evaluate(() => {
    function cleanHtml(root: Element): string {
      const clone = root.cloneNode(true) as Element;
      clone.querySelectorAll("script, style, iframe, object, embed, form, button, textarea, input, nav").forEach((node) => node.remove());
      clone.querySelectorAll("*").forEach((element) => {
        Array.from(element.attributes).forEach((attr) => {
          const name = attr.name.toLowerCase();
          const keep = ["href", "src", "alt", "title", "colspan", "rowspan"].includes(name);
          if (name.startsWith("on") || !keep) element.removeAttribute(attr.name);
          if ((name === "href" || name === "src") && /^javascript:/i.test(attr.value)) element.removeAttribute(attr.name);
        });
      });
      return clone.innerHTML.trim();
    }

    function textToHtml(text: string): string {
      const escaped = text
        .replace(/&/g, "&amp;")
        .replace(/</g, "&lt;")
        .replace(/>/g, "&gt;");
      return escaped
        .split(/\n{2,}/)
        .map((part) => `<p>${part.replace(/\n/g, "<br>")}</p>`)
        .join("");
    }

    const title =
      document.querySelector("h1")?.textContent?.trim() ||
      document.title.replace(/\s*[|-]\s*ChatGPT\s*$/i, "").trim() ||
      "ChatGPT Share";

    const roleNodes = Array.from(document.querySelectorAll("[data-message-author-role]"));
    const articleNodes = roleNodes.length ? [] : Array.from(document.querySelectorAll("main article"));
    const nodes = roleNodes.length ? roleNodes : articleNodes;
    const messages = nodes
      .map((node, index) => {
        const roleAttr = node.getAttribute("data-message-author-role") || "";
        const role =
          roleAttr === "user" || roleAttr === "assistant" || roleAttr === "system"
            ? roleAttr
            : index % 2 === 0
              ? "user"
              : "assistant";
        const contentRoot =
          node.querySelector(".markdown") ||
          node.querySelector("[data-message-content]") ||
          node;
        const text = (contentRoot.textContent || "").replace(/\s+\n/g, "\n").trim();
        const html = cleanHtml(contentRoot) || textToHtml(text);
        return { id: `m-${index + 1}`, role, text, html };
      })
      .filter((message) => message.text.length > 0);

    const bodyText = document.body.textContent || "";
    if (/verify you are human|captcha|unusual activity/i.test(bodyText)) {
      throw new Error("ChatGPT verification or CAPTCHA page was shown.");
    }

    return { title, messages };
  });

  await context.close();

  if (!extracted.messages.length) {
    throw new Error("Unsupported ChatGPT share page: no conversation messages were found.");
  }

  return extracted as ExtractedConversation;
}

async function generateScreenshotVariant(slug: string, variant: ScreenshotVariant): Promise<void> {
  const browser = await chromium.launch({ headless: true });
  try {
    const isMobile = variant.device === "mobile";
    const context = await browser.newContext({
      viewport: isMobile ? { width: 430, height: 932 } : { width: 1360, height: 900 },
      deviceScaleFactor: 2,
      colorScheme: variant.theme,
      isMobile
    });
    const page = await context.newPage();
    const fileUrl = `file://${artifactIndexPath(slug)}?theme=${variant.theme}`;
    await page.goto(fileUrl, { waitUntil: "load", timeout: 60_000 });
    const height = await page.evaluate(() => Math.ceil(document.documentElement.scrollHeight));
    if (height > config.maxScreenshotHeight) {
      throw new Error(`Screenshot height ${height}px exceeds limit ${config.maxScreenshotHeight}px.`);
    }
    await fs.ensureDir(screenshotsDir(slug));
    await page.screenshot({
      path: screenshotPath(slug, `${variant.device}-${variant.theme}`),
      fullPage: true,
      type: "png"
    });
    await context.close();
  } finally {
    await browser.close();
  }
}

async function generateScreenshots(slug: string): Promise<{ generated: string[]; failed: string[] }> {
  const variants: ScreenshotVariant[] = [
    { device: "desktop", theme: "light" },
    { device: "desktop", theme: "dark" },
    { device: "mobile", theme: "light" },
    { device: "mobile", theme: "dark" }
  ];
  const generated: string[] = [];
  const failed: string[] = [];
  for (const variant of variants) {
    const name = `${variant.device}-${variant.theme}`;
    try {
      await generateScreenshotVariant(slug, variant);
      generated.push(name);
    } catch (error) {
      failed.push(`${name}: ${error instanceof Error ? error.message : String(error)}`);
    }
  }
  return { generated, failed };
}

export async function runCapture(row: CaptureRow): Promise<void> {
  let browser: Browser | null = null;
  const fallbackSlug = store.uniqueSlug(slugify(row.requested_slug || `chatgpt-share-${row.id.slice(0, 8)}`));
  try {
    store.updateStatus(row.id, "capturing");
    browser = await chromium.launch({ headless: true });
    const extracted = await extractConversation(browser, row.source_url);
    await browser.close();
    browser = null;

    const slug = store.uniqueSlug(slugify(row.requested_slug || extracted.title || fallbackSlug));
    const conversation: NormalizedConversation = {
      provider: "chatgpt",
      title: extracted.title || "ChatGPT Share",
      sourceUrl: row.source_url,
      capturedAt: new Date().toISOString(),
      messages: extracted.messages
    };

    store.updateStatus(row.id, "rendering");
    const assetResult = await localizeAssets(conversation, slug);
    await writeConversationArtifact(slug, conversation);
    const screenshots = await generateScreenshots(slug);

    store.completeCapture(row.id, {
      slug,
      title: conversation.title,
      dataJson: JSON.stringify(conversation),
      metadataJson: JSON.stringify({
        assetFailures: assetResult.failures,
        screenshots
      })
    });
  } catch (error) {
    if (browser) await browser.close().catch(() => undefined);
    const current = store.getCapture(row.id) || row;
    const slug = current.slug || fallbackSlug;
    store.failCapture(row.id, {
      slug,
      title: current.title || "Capture failed",
      errorPublic: publicErrorFor(error),
      errorDetail: error instanceof Error ? `${error.message}\n${error.stack || ""}` : String(error),
      metadataJson: JSON.stringify({ failedAt: new Date().toISOString() })
    });
    const failedRow = store.getCapture(row.id);
    if (failedRow) await writeFailureArtifact(failedRow);
  }
}

export function newCaptureId(): string {
  return nanoid(12);
}