Initial aishare service
This commit is contained in:
Codex
+278
@@ -0,0 +1,278 @@
|
||||
import path from "node:path";
|
||||
import fs from "fs-extra";
|
||||
import * as cheerio from "cheerio";
|
||||
import { chromium, type Browser } from "@playwright/test";
|
||||
import { nanoid } from "nanoid";
|
||||
import { store } from "./db.js";
|
||||
import { config } from "./config.js";
|
||||
import { artifactAssetsDir, artifactIndexPath, safeAssetName, screenshotPath, screenshotsDir } from "./artifacts.js";
|
||||
import { slugify } from "./slug.js";
|
||||
import { writeConversationArtifact, writeFailureArtifact } from "./render.js";
|
||||
import type { CaptureRow, NormalizedConversation, NormalizedMessage, ScreenshotVariant } from "./types.js";
|
||||
|
||||
interface ExtractedConversation {
|
||||
title: string;
|
||||
messages: NormalizedMessage[];
|
||||
}
|
||||
|
||||
function publicErrorFor(error: unknown): string {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (/captcha|verify|robot/i.test(message)) {
|
||||
return "ChatGPT asked for verification, so this public share could not be captured automatically.";
|
||||
}
|
||||
if (/timeout/i.test(message)) {
|
||||
return "The public ChatGPT share took too long to load.";
|
||||
}
|
||||
if (/unsupported|message/i.test(message)) {
|
||||
return "This public ChatGPT share could not be converted because its page format is not supported yet.";
|
||||
}
|
||||
return "This public ChatGPT share could not be captured.";
|
||||
}
|
||||
|
||||
function timeoutSignal(ms: number): AbortSignal {
|
||||
const controller = new AbortController();
|
||||
setTimeout(() => controller.abort(), ms).unref();
|
||||
return controller.signal;
|
||||
}
|
||||
|
||||
async function downloadAsset(assetUrl: string, slug: string): Promise<string | null> {
|
||||
const response = await fetch(assetUrl, {
|
||||
signal: timeoutSignal(20_000),
|
||||
headers: {
|
||||
"user-agent": "aishare/0.1"
|
||||
}
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
const contentType = response.headers.get("content-type");
|
||||
if (!contentType?.startsWith("image/")) return null;
|
||||
const bytes = Buffer.from(await response.arrayBuffer());
|
||||
if (bytes.byteLength > 20 * 1024 * 1024) return null;
|
||||
const filename = safeAssetName(assetUrl, contentType, bytes);
|
||||
await fs.ensureDir(artifactAssetsDir(slug));
|
||||
await fs.writeFile(path.join(artifactAssetsDir(slug), filename), bytes);
|
||||
return `/aishare/${slug}/assets/${filename}`;
|
||||
}
|
||||
|
||||
async function localizeAssets(conversation: NormalizedConversation, slug: string): Promise<{ failures: string[] }> {
|
||||
const failures: string[] = [];
|
||||
for (const message of conversation.messages) {
|
||||
const $ = cheerio.load(message.html, {}, false);
|
||||
const images = $("img").toArray();
|
||||
for (const image of images) {
|
||||
const current = $(image).attr("src");
|
||||
if (!current) continue;
|
||||
let absolute: string;
|
||||
try {
|
||||
absolute = new URL(current, conversation.sourceUrl).toString();
|
||||
} catch {
|
||||
$(image).removeAttr("src");
|
||||
failures.push(current);
|
||||
continue;
|
||||
}
|
||||
if (!absolute.startsWith("https://")) {
|
||||
$(image).removeAttr("src");
|
||||
failures.push(absolute);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const local = await downloadAsset(absolute, slug);
|
||||
if (local) {
|
||||
$(image).attr("src", local);
|
||||
} else {
|
||||
$(image).removeAttr("src");
|
||||
failures.push(absolute);
|
||||
}
|
||||
} catch {
|
||||
$(image).removeAttr("src");
|
||||
failures.push(absolute);
|
||||
}
|
||||
}
|
||||
message.html = $.html();
|
||||
}
|
||||
return { failures };
|
||||
}
|
||||
|
||||
async function extractConversation(browser: Browser, sourceUrl: string): Promise<ExtractedConversation> {
|
||||
const context = await browser.newContext({
|
||||
viewport: { width: 1360, height: 900 },
|
||||
deviceScaleFactor: 1,
|
||||
locale: "en-US"
|
||||
});
|
||||
const page = await context.newPage();
|
||||
page.setDefaultTimeout(config.captureTimeoutMs);
|
||||
await page.goto(sourceUrl, { waitUntil: "domcontentloaded", timeout: config.captureTimeoutMs });
|
||||
await page.waitForLoadState("networkidle", { timeout: Math.min(config.captureTimeoutMs, 60_000) }).catch(() => undefined);
|
||||
await page.waitForTimeout(2500);
|
||||
|
||||
const extracted = await page.evaluate(() => {
|
||||
function cleanHtml(root: Element): string {
|
||||
const clone = root.cloneNode(true) as Element;
|
||||
clone.querySelectorAll("script, style, iframe, object, embed, form, button, textarea, input, nav").forEach((node) => node.remove());
|
||||
clone.querySelectorAll("*").forEach((element) => {
|
||||
Array.from(element.attributes).forEach((attr) => {
|
||||
const name = attr.name.toLowerCase();
|
||||
const keep = ["href", "src", "alt", "title", "colspan", "rowspan"].includes(name);
|
||||
if (name.startsWith("on") || !keep) element.removeAttribute(attr.name);
|
||||
if ((name === "href" || name === "src") && /^javascript:/i.test(attr.value)) element.removeAttribute(attr.name);
|
||||
});
|
||||
});
|
||||
return clone.innerHTML.trim();
|
||||
}
|
||||
|
||||
function textToHtml(text: string): string {
|
||||
const escaped = text
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">");
|
||||
return escaped
|
||||
.split(/\n{2,}/)
|
||||
.map((part) => `<p>${part.replace(/\n/g, "<br>")}</p>`)
|
||||
.join("");
|
||||
}
|
||||
|
||||
const title =
|
||||
document.querySelector("h1")?.textContent?.trim() ||
|
||||
document.title.replace(/\s*[|-]\s*ChatGPT\s*$/i, "").trim() ||
|
||||
"ChatGPT Share";
|
||||
|
||||
const roleNodes = Array.from(document.querySelectorAll("[data-message-author-role]"));
|
||||
const articleNodes = roleNodes.length ? [] : Array.from(document.querySelectorAll("main article"));
|
||||
const nodes = roleNodes.length ? roleNodes : articleNodes;
|
||||
const messages = nodes
|
||||
.map((node, index) => {
|
||||
const roleAttr = node.getAttribute("data-message-author-role") || "";
|
||||
const role =
|
||||
roleAttr === "user" || roleAttr === "assistant" || roleAttr === "system"
|
||||
? roleAttr
|
||||
: index % 2 === 0
|
||||
? "user"
|
||||
: "assistant";
|
||||
const contentRoot =
|
||||
node.querySelector(".markdown") ||
|
||||
node.querySelector("[data-message-content]") ||
|
||||
node;
|
||||
const text = (contentRoot.textContent || "").replace(/\s+\n/g, "\n").trim();
|
||||
const html = cleanHtml(contentRoot) || textToHtml(text);
|
||||
return { id: `m-${index + 1}`, role, text, html };
|
||||
})
|
||||
.filter((message) => message.text.length > 0);
|
||||
|
||||
const bodyText = document.body.textContent || "";
|
||||
if (/verify you are human|captcha|unusual activity/i.test(bodyText)) {
|
||||
throw new Error("ChatGPT verification or CAPTCHA page was shown.");
|
||||
}
|
||||
|
||||
return { title, messages };
|
||||
});
|
||||
|
||||
await context.close();
|
||||
|
||||
if (!extracted.messages.length) {
|
||||
throw new Error("Unsupported ChatGPT share page: no conversation messages were found.");
|
||||
}
|
||||
|
||||
return extracted as ExtractedConversation;
|
||||
}
|
||||
|
||||
async function generateScreenshotVariant(slug: string, variant: ScreenshotVariant): Promise<void> {
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
try {
|
||||
const isMobile = variant.device === "mobile";
|
||||
const context = await browser.newContext({
|
||||
viewport: isMobile ? { width: 430, height: 932 } : { width: 1360, height: 900 },
|
||||
deviceScaleFactor: 2,
|
||||
colorScheme: variant.theme,
|
||||
isMobile
|
||||
});
|
||||
const page = await context.newPage();
|
||||
const fileUrl = `file://${artifactIndexPath(slug)}?theme=${variant.theme}`;
|
||||
await page.goto(fileUrl, { waitUntil: "load", timeout: 60_000 });
|
||||
const height = await page.evaluate(() => Math.ceil(document.documentElement.scrollHeight));
|
||||
if (height > config.maxScreenshotHeight) {
|
||||
throw new Error(`Screenshot height ${height}px exceeds limit ${config.maxScreenshotHeight}px.`);
|
||||
}
|
||||
await fs.ensureDir(screenshotsDir(slug));
|
||||
await page.screenshot({
|
||||
path: screenshotPath(slug, `${variant.device}-${variant.theme}`),
|
||||
fullPage: true,
|
||||
type: "png"
|
||||
});
|
||||
await context.close();
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function generateScreenshots(slug: string): Promise<{ generated: string[]; failed: string[] }> {
|
||||
const variants: ScreenshotVariant[] = [
|
||||
{ device: "desktop", theme: "light" },
|
||||
{ device: "desktop", theme: "dark" },
|
||||
{ device: "mobile", theme: "light" },
|
||||
{ device: "mobile", theme: "dark" }
|
||||
];
|
||||
const generated: string[] = [];
|
||||
const failed: string[] = [];
|
||||
for (const variant of variants) {
|
||||
const name = `${variant.device}-${variant.theme}`;
|
||||
try {
|
||||
await generateScreenshotVariant(slug, variant);
|
||||
generated.push(name);
|
||||
} catch (error) {
|
||||
failed.push(`${name}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
return { generated, failed };
|
||||
}
|
||||
|
||||
export async function runCapture(row: CaptureRow): Promise<void> {
|
||||
let browser: Browser | null = null;
|
||||
const fallbackSlug = store.uniqueSlug(slugify(row.requested_slug || `chatgpt-share-${row.id.slice(0, 8)}`));
|
||||
try {
|
||||
store.updateStatus(row.id, "capturing");
|
||||
browser = await chromium.launch({ headless: true });
|
||||
const extracted = await extractConversation(browser, row.source_url);
|
||||
await browser.close();
|
||||
browser = null;
|
||||
|
||||
const slug = store.uniqueSlug(slugify(row.requested_slug || extracted.title || fallbackSlug));
|
||||
const conversation: NormalizedConversation = {
|
||||
provider: "chatgpt",
|
||||
title: extracted.title || "ChatGPT Share",
|
||||
sourceUrl: row.source_url,
|
||||
capturedAt: new Date().toISOString(),
|
||||
messages: extracted.messages
|
||||
};
|
||||
|
||||
store.updateStatus(row.id, "rendering");
|
||||
const assetResult = await localizeAssets(conversation, slug);
|
||||
await writeConversationArtifact(slug, conversation);
|
||||
const screenshots = await generateScreenshots(slug);
|
||||
|
||||
store.completeCapture(row.id, {
|
||||
slug,
|
||||
title: conversation.title,
|
||||
dataJson: JSON.stringify(conversation),
|
||||
metadataJson: JSON.stringify({
|
||||
assetFailures: assetResult.failures,
|
||||
screenshots
|
||||
})
|
||||
});
|
||||
} catch (error) {
|
||||
if (browser) await browser.close().catch(() => undefined);
|
||||
const current = store.getCapture(row.id) || row;
|
||||
const slug = current.slug || fallbackSlug;
|
||||
store.failCapture(row.id, {
|
||||
slug,
|
||||
title: current.title || "Capture failed",
|
||||
errorPublic: publicErrorFor(error),
|
||||
errorDetail: error instanceof Error ? `${error.message}\n${error.stack || ""}` : String(error),
|
||||
metadataJson: JSON.stringify({ failedAt: new Date().toISOString() })
|
||||
});
|
||||
const failedRow = store.getCapture(row.id);
|
||||
if (failedRow) await writeFailureArtifact(failedRow);
|
||||
}
|
||||
}
|
||||
|
||||
export function newCaptureId(): string {
|
||||
return nanoid(12);
|
||||
}
|
||||
Reference in New Issue
Block a user