Normalize MathML in captured shares

2026-07-08 09:28:30 +00:00
parent 8734bc1f07
commit 1f4d1dde1b
2 changed files with 182 additions and 2 deletions
@@ -125,6 +125,106 @@ function normalizeCopiedHtml(html: string): string {
    return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
  }

+  function comparableMathText(value: string): string {
+    return value
+      .replace(/\\div/g, "÷")
+      .replace(/\s+/g, "")
+      .trim();
+  }
+
+  function trimDuplicateMathFallback(fallback: string, expression: string): string | null {
+    const target = comparableMathText(expression);
+    if (!target) return null;
+
+    let consumed = "";
+    let index = 0;
+    while (index < fallback.length && consumed.length < target.length) {
+      const next = fallback.startsWith("\\div", index) ? "÷" : fallback[index];
+      const width = fallback.startsWith("\\div", index) ? 4 : 1;
+      index += width;
+      if (/\s/.test(next)) continue;
+      consumed += next;
+    }
+
+    if (consumed === target) return fallback.slice(index);
+
+    const leading = fallback.match(/^\s*/)?.[0] || "";
+    const rest = fallback.slice(leading.length);
+    const fallbackRun = rest.match(/^[^\s<]+/)?.[0] || "";
+    const mathOperators = /[=∑∏√±×⋅*/^_≥≤<>−+÷]/;
+    if (fallbackRun.length >= 3 && mathOperators.test(fallbackRun) && mathOperators.test(target)) {
+      return `${leading}${rest.slice(fallbackRun.length)}`;
+    }
+
+    const firstLine = rest.split(/\n/, 1)[0] || "";
+    const compactLine = comparableMathText(firstLine);
+    if (compactLine.length >= 8 && mathOperators.test(compactLine) && mathOperators.test(target)) {
+      const available = new Map<string, number>();
+      for (const char of target) available.set(char, (available.get(char) || 0) + 1);
+      let shared = 0;
+      for (const char of compactLine) {
+        const count = available.get(char) || 0;
+        if (count <= 0) continue;
+        available.set(char, count - 1);
+        shared += 1;
+      }
+      if (shared / Math.min(target.length, compactLine.length) >= 0.6) {
+        return `${leading}${rest.slice(firstLine.length)}`;
+      }
+    }
+
+    return null;
+  }
+
+  function normalizeMathMl(document: cheerio.CheerioAPI): void {
+    type DomNode = {
+      type?: string;
+      name?: string;
+      data?: string;
+      children?: DomNode[];
+      parent?: DomNode;
+    };
+
+    function visibleMathText(math: DomNode): string {
+      const visibleMathNode = document(math as never).clone();
+      visibleMathNode.find("annotation, annotation-xml").remove();
+      return visibleMathNode.text();
+    }
+
+    function trimTextNode(node: DomNode | undefined, expression: string): boolean {
+      if (!node || node.type !== "text" || typeof node.data !== "string") return false;
+      const trimmed = trimDuplicateMathFallback(node.data, expression);
+      if (trimmed === null) return false;
+      node.data = trimmed;
+      return true;
+    }
+
+    function trimSimpleFallbackElement(node: DomNode | undefined, expression: string): boolean {
+      if (!node || node.type !== "tag" || node.name !== "span") return false;
+      const children = node.children || [];
+      if (children.length !== 1 || children[0]?.type !== "text") return false;
+      return trimTextNode(children[0], expression);
+    }
+
+    function normalizeChildList(children: DomNode[]): void {
+      for (let index = 0; index < children.length; index += 1) {
+        const child = children[index];
+        if (child?.children) normalizeChildList(child.children);
+        if (child?.type !== "tag" || child.name !== "math") continue;
+
+        const visibleMath = visibleMathText(child);
+        const next = children[index + 1];
+        trimTextNode(next, visibleMath) || trimSimpleFallbackElement(next, visibleMath);
+      }
+    }
+
+    document("math").find("annotation, annotation-xml").remove();
+    normalizeChildList(document.root()[0].children as DomNode[]);
+    document("math").each((_, math) => {
+      document(math).find("annotation, annotation-xml").remove();
+    });
+  }
+
  $("pre")
    .toArray()
    .filter((pre) => $(pre).parents("pre").length === 0)
@@ -167,6 +267,7 @@ function normalizeCopiedHtml(html: string): string {

  $("span").each((_, span) => {
    const element = $(span);
+    if (element.closest("a[data-citation]").length > 0) return;
    if (Object.keys(span.attribs || {}).length === 0) element.replaceWith(element.contents());
  });

@@ -177,8 +278,17 @@ function normalizeCopiedHtml(html: string): string {
    );
  });

-  $(":empty").not("br,img,hr").remove();
-  return $.html().trim();
+  normalizeMathMl($);
+
+  $(":empty")
+    .not("br,img,hr")
+    .filter((_, element) => $(element).parents("math").length === 0)
+    .remove();
+  normalizeMathMl($);
+
+  const finalDocument = cheerio.load($.html(), {}, false);
+  normalizeMathMl(finalDocument);
+  return finalDocument.html().trim();
 }

 function findChromiumExecutable(): string {
@@ -343,6 +453,73 @@ async function extractConversation(sourceUrl: string): Promise<ExtractedConversa
          if ((name === "href" || name === "src") && /^javascript:/i.test(attr.value)) element.removeAttribute(attr.name);
        });
      });
+
+      function comparableMathText(value: string): string {
+        return value.replace(/\\div/g, "÷").replace(/\s+/g, "").trim();
+      }
+
+      function trimDuplicateMathFallback(fallback: string, expression: string): string | null {
+        const target = comparableMathText(expression);
+        if (!target) return null;
+        let consumed = "";
+        let index = 0;
+        while (index < fallback.length && consumed.length < target.length) {
+          const next = fallback.startsWith("\\div", index) ? "÷" : fallback[index];
+          const width = fallback.startsWith("\\div", index) ? 4 : 1;
+          index += width;
+          if (/\s/.test(next)) continue;
+          consumed += next;
+        }
+        if (consumed === target) return fallback.slice(index);
+
+        const leading = fallback.match(/^\s*/)?.[0] || "";
+        const rest = fallback.slice(leading.length);
+        const fallbackRun = rest.match(/^[^\s<]+/)?.[0] || "";
+        const mathOperators = /[=∑∏√±×⋅*/^_≥≤<>−+÷]/;
+        if (fallbackRun.length >= 3 && mathOperators.test(fallbackRun) && mathOperators.test(target)) {
+          return `${leading}${rest.slice(fallbackRun.length)}`;
+        }
+
+        const firstLine = rest.split(/\n/, 1)[0] || "";
+        const compactLine = comparableMathText(firstLine);
+        if (compactLine.length >= 8 && mathOperators.test(compactLine) && mathOperators.test(target)) {
+          const available = new Map<string, number>();
+          for (const char of target) available.set(char, (available.get(char) || 0) + 1);
+          let shared = 0;
+          for (const char of compactLine) {
+            const count = available.get(char) || 0;
+            if (count <= 0) continue;
+            available.set(char, count - 1);
+            shared += 1;
+          }
+          if (shared / Math.min(target.length, compactLine.length) >= 0.6) {
+            return `${leading}${rest.slice(firstLine.length)}`;
+          }
+        }
+
+        return null;
+      }
+
+      function trimMathFallbackNode(node: ChildNode | null, expression: string): void {
+        if (node?.nodeType === Node.TEXT_NODE) {
+          const trimmed = trimDuplicateMathFallback(node.textContent || "", expression);
+          if (trimmed !== null) node.textContent = trimmed;
+          return;
+        }
+        if (!(node instanceof HTMLElement) || node.tagName.toLowerCase() !== "span") return;
+        if (node.childNodes.length !== 1 || node.firstChild?.nodeType !== Node.TEXT_NODE) return;
+        const trimmed = trimDuplicateMathFallback(node.textContent || "", expression);
+        if (trimmed !== null) node.textContent = trimmed;
+      }
+
+      clone.querySelectorAll("math").forEach((math) => {
+        const visible = math.cloneNode(true) as Element;
+        visible.querySelectorAll("annotation, annotation-xml").forEach((node) => node.remove());
+        const visibleText = visible.textContent || "";
+        math.querySelectorAll("annotation, annotation-xml").forEach((node) => node.remove());
+        trimMathFallbackNode(math.nextSibling, visibleText);
+      });
+
      return clone.innerHTML.trim();
    }

@@ -110,6 +110,9 @@ h1 { margin: 0; font-size: clamp(28px, 5vw, 44px); line-height: 1.12; letter-spa
 .content th, .content td { border: 1px solid var(--border); padding: 8px 10px; text-align: left; vertical-align: top; }
 .content blockquote { border-left: 3px solid var(--border); margin: 1em 0; padding-left: 14px; color: var(--muted); }
 .content img { max-width: 100%; height: auto; border-radius: 8px; }
+.content math { color: inherit; font-family: math, "STIX Two Math", "Cambria Math", "Noto Sans Math", serif; font-size: 1.02em; }
+.content > math { display: block; max-width: 100%; margin: 0.95em 0; overflow-x: auto; overflow-y: hidden; }
+.content p math, .content li math { vertical-align: -0.08em; }
 .content a[data-citation] { display: inline-flex; align-items: center; gap: 5px; max-width: min(100%, 260px); min-height: 24px; margin: 0 2px; padding: 2px 8px 2px 5px; border: 1px solid var(--border); border-radius: 999px; background: var(--soft); color: var(--text); font-size: 0.88em; line-height: 1.25; white-space: nowrap; text-decoration: none; vertical-align: -5px; }
 .content a[data-citation]:hover { background: color-mix(in srgb, var(--soft) 82%, var(--text) 18%); }
 .content a[data-citation] img { width: 16px; height: 16px; max-width: 16px; max-height: 16px; border-radius: 50%; object-fit: cover; flex: 0 0 auto; }