Extract ChatGPT shares through raw CDP

src/capture.ts

@@ -1,3 +1,7 @@
+import { spawn, type ChildProcess } from "node:child_process";
+import nodeFs from "node:fs";
+import net from "node:net";
+import os from "node:os";
 import path from "node:path";
 import fs from "fs-extra";
 import * as cheerio from "cheerio";
@@ -92,15 +96,104 @@ async function localizeAssets(conversation: NormalizedConversation, slug: string
   return { failures };
 }
 
-async function extractConversation(browser: Browser, sourceUrl: string): Promise<ExtractedConversation> {
+function findChromiumExecutable(): string {
-  const context = await browser.newContext({
+  const preferred = chromium.executablePath();
-    viewport: { width: 1360, height: 900 },
+  if (nodeFs.existsSync(preferred)) return preferred;
-    deviceScaleFactor: 1,
+
-    locale: "en-US"
+  const browsersRoot = process.env.PLAYWRIGHT_BROWSERS_PATH || path.dirname(path.dirname(preferred));
+  const headlessShell = nodeFs
+    .readdirSync(browsersRoot)
+    .filter((entry) => entry.startsWith("chromium_headless_shell-"))
+    .sort()
+    .reverse()
+    .map((entry) => path.join(browsersRoot, entry, "chrome-headless-shell-linux64", "chrome-headless-shell"))
+    .find((candidate) => nodeFs.existsSync(candidate));
+
+  if (headlessShell) return headlessShell;
+  throw new Error(`No Chromium executable found. Tried ${preferred} and ${browsersRoot}/chromium_headless_shell-*.`);
+}
+
+async function getFreeLocalPort(): Promise<number> {
+  return await new Promise((resolve, reject) => {
+    const server = net.createServer();
+    server.once("error", reject);
+    server.listen(0, "127.0.0.1", () => {
+      const address = server.address();
+      server.close(() => {
+        if (address && typeof address === "object") {
+          resolve(address.port);
+        } else {
+          reject(new Error("Could not allocate a local CDP port."));
+        }
+      });
+    });
   });
-  const page = await context.newPage();
+}
+
+async function waitForCdp(port: number, process: ChildProcess): Promise<void> {
+  const deadline = Date.now() + Math.min(config.captureTimeoutMs, 30_000);
+  while (Date.now() < deadline) {
+    if (process.exitCode !== null) {
+      throw new Error(`Chromium exited before CDP became ready with code ${process.exitCode}.`);
+    }
+    const response = await fetch(`http://127.0.0.1:${port}/json/version`).catch(() => null);
+    if (response?.ok) return;
+    await new Promise((resolve) => setTimeout(resolve, 250));
+  }
+  throw new Error("Chromium CDP endpoint did not become ready.");
+}
+
+async function stopProcess(process: ChildProcess): Promise<void> {
+  if (process.exitCode !== null || process.signalCode !== null) return;
+  process.kill("SIGTERM");
+  await new Promise<void>((resolve) => {
+    const timer = setTimeout(() => {
+      if (process.exitCode === null && process.signalCode === null) process.kill("SIGKILL");
+      resolve();
+    }, 3000);
+    process.once("exit", () => {
+      clearTimeout(timer);
+      resolve();
+    });
+  });
+}
+
+async function extractConversation(sourceUrl: string): Promise<ExtractedConversation> {
+  const port = await getFreeLocalPort();
+  const userDataDir = path.join(os.tmpdir(), `aishare-capture-${nanoid()}`);
+  await fs.ensureDir(userDataDir);
+
+  const chromiumProcess = spawn(
+    findChromiumExecutable(),
+    [
+      "--headless",
+      "--no-sandbox",
+      "--disable-dev-shm-usage",
+      "--disable-gpu",
+      "--hide-scrollbars",
+      "--mute-audio",
+      "--window-size=1360,900",
+      "--remote-debugging-address=127.0.0.1",
+      `--remote-debugging-port=${port}`,
+      `--user-data-dir=${userDataDir}`,
+      sourceUrl
+    ],
+    { stdio: "ignore" }
+  );
+
+  let browser: Browser | null = null;
+  try {
+    await waitForCdp(port, chromiumProcess);
+    browser = await chromium.connectOverCDP(`http://127.0.0.1:${port}`, { timeout: config.captureTimeoutMs });
+    const context = browser.contexts()[0];
+    let page = context.pages().find((candidate) => candidate.url().includes("/share/")) || context.pages()[0];
+    for (let attempt = 0; !page && attempt < 100; attempt += 1) {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      page = context.pages().find((candidate) => candidate.url().includes("/share/")) || context.pages()[0];
+    }
+    if (!page) throw new Error("Chromium CDP page target was not created.");
+    await page.setViewportSize({ width: 1360, height: 900 }).catch(() => undefined);
   page.setDefaultTimeout(config.captureTimeoutMs);
-  await page.goto(sourceUrl, { waitUntil: "commit", timeout: config.captureTimeoutMs });
   await page
     .waitForFunction(
       () => {
@@ -178,13 +271,16 @@ async function extractConversation(browser: Browser, sourceUrl: string): Promise
     return { title, messages };
   });
 
-  await context.close();
-
   if (!extracted.messages.length) {
     throw new Error("Unsupported ChatGPT share page: no conversation messages were found.");
   }
 
   return extracted as ExtractedConversation;
+  } finally {
+    if (browser) await browser.close().catch(() => undefined);
+    await stopProcess(chromiumProcess).catch(() => undefined);
+    await fs.remove(userDataDir).catch(() => undefined);
+  }
 }
 
 async function generateScreenshotVariant(slug: string, variant: ScreenshotVariant): Promise<void> {
@@ -238,14 +334,10 @@ async function generateScreenshots(slug: string): Promise<{ generated: string[];
 }
 
 export async function runCapture(row: CaptureRow): Promise<void> {
-  let browser: Browser | null = null;
   const fallbackSlug = store.uniqueSlug(slugify(row.requested_slug || `chatgpt-share-${row.id.slice(0, 8)}`));
   try {
     store.updateStatus(row.id, "capturing");
-    browser = await chromium.launch({ headless: true });
+    const extracted = await extractConversation(row.source_url);
-    const extracted = await extractConversation(browser, row.source_url);
-    await browser.close();
-    browser = null;
 
     const slug = store.uniqueSlug(slugify(row.requested_slug || extracted.title || fallbackSlug));
     const conversation: NormalizedConversation = {
@@ -271,7 +363,6 @@ export async function runCapture(row: CaptureRow): Promise<void> {
       })
     });
   } catch (error) {
-    if (browser) await browser.close().catch(() => undefined);
     const current = store.getCapture(row.id) || row;
     const slug = current.slug || fallbackSlug;
     store.failCapture(row.id, {