DoLuEdu-WordFileParse/scripts/omml_mathml2latex/utils/xml_utils.py

"""
XML processing utilities for MathML and OMML handling.

This module provides helper functions for working with XML/MathML/OMML documents,
including namespace handling, text extraction, and HTML entity decoding.
"""

import re
import html
from typing import List
from lxml import etree as ET


def local_name(tag: str) -> str:
    """Extract local tag name without namespace.

    Args:
        tag: XML tag name, possibly with namespace prefix like "{http://...}tagname"

    Returns:
        Local tag name without namespace

    Example:
        >>> local_name("{http://www.w3.org/1998/Math/MathML}mi")
        'mi'
        >>> local_name("mi")
        'mi'
    """
    if "}" in tag:
        return tag.split("}", 1)[1]
    return tag


def text_content(node: ET._Element) -> str:
    """Flatten text under a node (best-effort).

    NOTE: Does NOT strip() to preserve space-only content like NBSP and regular spaces.
    Spaces are semantically meaningful in mathematical notation.

    However, DOES remove invisible formatting characters like function application (\u2061)
    and normalizes whitespace (converts newlines/tabs to single spaces).

    Args:
        node: XML element to extract text from

    Returns:
        Concatenated text content from node and all its children

    Example:
        >>> from lxml import etree as ET
        >>> node = ET.fromstring('<mi>sin</mi>')
        >>> text_content(node)
        'sin'
    """
    texts: List[str] = []
    if node.text:
        texts.append(node.text)
    for child in node:
        texts.append(text_content(child))
        if child.tail:
            texts.append(child.tail)

    result = "".join(texts)

    # Remove invisible formatting characters that interfere with conversion
    # but preserve meaningful spaces (regular space, nbsp)
    invisible_chars = [
        '\u2061',  # FUNCTION APPLICATION (causes sin⁡ → \mathrm{sin})
        '\u2062',  # INVISIBLE TIMES
        '\u2063',  # INVISIBLE SEPARATOR
        '\u2064',  # INVISIBLE PLUS
        '\uFEFF',  # ZERO WIDTH NO-BREAK SPACE
        '\u200B',  # ZERO WIDTH SPACE
        '\u200C',  # ZERO WIDTH NON-JOINER
        '\u200D',  # ZERO WIDTH JOINER
    ]

    for char in invisible_chars:
        result = result.replace(char, '')

    # Normalize whitespace: Remove newlines and tabs completely
    # This fixes both multiline fractions like \dfrac{\n  2\n}
    # AND prevents formatting whitespace from JSON/XML from being converted to LaTeX spacing
    # Preserve NBSP (\xa0) as it's semantically meaningful
    result = re.sub(r'[\n\r\t]+', '', result)

    # Collapse multiple consecutive regular spaces into one
    # This prevents "         2       " → should be " 2 " or just "2"
    # But don't collapse NBSP (\xa0) as it's semantically meaningful
    result = re.sub(r' +', ' ', result)

    # Strip leading/trailing spaces that come from XML formatting
    # These are not semantically meaningful - they're just from pretty-printed XML
    # True semantic spaces (like in operators) are preserved within the content
    result = result.strip()

    return result


def decode_html_entities(text: str) -> str:
    """Decode HTML entities in text, particularly &quot; from web page copies.

    IMPORTANT: This function is designed for HTML content, NOT for XML/MathML!
    DO NOT use this on XML/MathML strings before XML parsing, as it will break
    valid XML numeric character references like &#x003C; which XML parsers
    handle natively.

    Args:
        text: Text possibly containing HTML entities

    Returns:
        Text with HTML named entities decoded (numeric entities preserved for XML)

    Example:
        >>> decode_html_entities('&lt;math&gt; &amp; &quot;test&quot;')
        '<math> & "test"'
        >>> decode_html_entities('<mo>&#x003C;</mo>')  # Preserves numeric entities
        '<mo>&#x003C;</mo>'
    """
    # CRITICAL FIX: Do NOT use html.unescape() on XML/MathML content!
    # html.unescape() decodes numeric character references like &#x003C; to <
    # which breaks XML parsing when < is inside element content.
    #
    # Instead, only decode common HTML named entities that might appear
    # in web page copies, and leave numeric entities for XML parser to handle.

    # Handle specific named entities that might appear in HTML copies
    # but preserve numeric entities (&#x...;, &#...;) for XML parsing
    entity_map = {
        '&quot;': '"',
        '&amp;': '&',
        '&lt;': '<',
        '&gt;': '>',
        '&apos;': "'",
        '&nbsp;': '\xa0',  # Non-breaking space
    }

    for entity, char in entity_map.items():
        text = text.replace(entity, char)

    return text


def extract_math_equations(text: str) -> List[str]:
    """Extract multiple MathML equations from text starting with <math tag.

    Args:
        text: Text possibly containing one or more MathML equations

    Returns:
        List of MathML equation strings (empty list if none found)

    Example:
        >>> text = '<math><mi>x</mi></math> and <math><mi>y</mi></math>'
        >>> equations = extract_math_equations(text)
        >>> len(equations)
        2
    """
    # CRITICAL FIX: Do NOT decode HTML entities before regex extraction!
    # The input is XML/MathML which contains numeric character references
    # like &#x003C; that are valid XML and must be preserved.
    # XML parsers (lxml) will handle entity decoding automatically.
    # Decoding them here would break valid XML like <mo>&#x003C;</mo> → <mo><</mo>
    #
    # Original line (REMOVED): text = decode_html_entities(text)

    equations = []

    # Find all <math> tags with their content
    # Use a more robust regex that handles nested tags and various formatting
    pattern = r'<math[^>]*>.*?</math>'
    matches = re.findall(pattern, text, re.DOTALL | re.IGNORECASE)

    for match in matches:
        # Clean up the equation
        equation = match.strip()
        if equation:
            # Decode HTML entities in the extracted equation
            # This handles cases like &quot; → " which are needed for valid XML
            equation = decode_html_entities(equation)
            equations.append(equation)

    # If no <math> tags found, check if the entire input is a single equation
    if not equations and text.strip().startswith('<math'):
        # Try to find the end of the math tag
        text = text.strip()
        # Simple case: if it looks like a complete math equation
        if '</math>' in text.lower():
            # Decode HTML entities before appending
            text = decode_html_entities(text)
            equations.append(text)
        else:
            # Try to fix incomplete equation by adding closing tag
            if not text.lower().endswith('</math>'):
                text += '</math>'
            # Decode HTML entities before appending
            text = decode_html_entities(text)
            equations.append(text)

    return equations


# Alias for backward compatibility (some code uses _local, some uses local_name)
_local = local_name
_text_content = text_content