""" XML processing utilities for MathML and OMML handling. This module provides helper functions for working with XML/MathML/OMML documents, including namespace handling, text extraction, and HTML entity decoding. """ import re import html from typing import List from lxml import etree as ET def local_name(tag: str) -> str: """Extract local tag name without namespace. Args: tag: XML tag name, possibly with namespace prefix like "{http://...}tagname" Returns: Local tag name without namespace Example: >>> local_name("{http://www.w3.org/1998/Math/MathML}mi") 'mi' >>> local_name("mi") 'mi' """ if "}" in tag: return tag.split("}", 1)[1] return tag def text_content(node: ET._Element) -> str: """Flatten text under a node (best-effort). NOTE: Does NOT strip() to preserve space-only content like NBSP and regular spaces. Spaces are semantically meaningful in mathematical notation. However, DOES remove invisible formatting characters like function application (\u2061) and normalizes whitespace (converts newlines/tabs to single spaces). Args: node: XML element to extract text from Returns: Concatenated text content from node and all its children Example: >>> from lxml import etree as ET >>> node = ET.fromstring('sin') >>> text_content(node) 'sin' """ texts: List[str] = [] if node.text: texts.append(node.text) for child in node: texts.append(text_content(child)) if child.tail: texts.append(child.tail) result = "".join(texts) # Remove invisible formatting characters that interfere with conversion # but preserve meaningful spaces (regular space, nbsp) invisible_chars = [ '\u2061', # FUNCTION APPLICATION (causes sin⁡ → \mathrm{sin}) '\u2062', # INVISIBLE TIMES '\u2063', # INVISIBLE SEPARATOR '\u2064', # INVISIBLE PLUS '\uFEFF', # ZERO WIDTH NO-BREAK SPACE '\u200B', # ZERO WIDTH SPACE '\u200C', # ZERO WIDTH NON-JOINER '\u200D', # ZERO WIDTH JOINER ] for char in invisible_chars: result = result.replace(char, '') # Normalize whitespace: Remove newlines and tabs completely # This fixes both multiline fractions like \dfrac{\n 2\n} # AND prevents formatting whitespace from JSON/XML from being converted to LaTeX spacing # Preserve NBSP (\xa0) as it's semantically meaningful result = re.sub(r'[\n\r\t]+', '', result) # Collapse multiple consecutive regular spaces into one # This prevents " 2 " → should be " 2 " or just "2" # But don't collapse NBSP (\xa0) as it's semantically meaningful result = re.sub(r' +', ' ', result) # Strip leading/trailing spaces that come from XML formatting # These are not semantically meaningful - they're just from pretty-printed XML # True semantic spaces (like in operators) are preserved within the content result = result.strip() return result def decode_html_entities(text: str) -> str: """Decode HTML entities in text, particularly " from web page copies. IMPORTANT: This function is designed for HTML content, NOT for XML/MathML! DO NOT use this on XML/MathML strings before XML parsing, as it will break valid XML numeric character references like < which XML parsers handle natively. Args: text: Text possibly containing HTML entities Returns: Text with HTML named entities decoded (numeric entities preserved for XML) Example: >>> decode_html_entities('<math> & "test"') ' & "test"' >>> decode_html_entities('<') # Preserves numeric entities '<' """ # CRITICAL FIX: Do NOT use html.unescape() on XML/MathML content! # html.unescape() decodes numeric character references like < to < # which breaks XML parsing when < is inside element content. # # Instead, only decode common HTML named entities that might appear # in web page copies, and leave numeric entities for XML parser to handle. # Handle specific named entities that might appear in HTML copies # but preserve numeric entities (&#x...;, &#...;) for XML parsing entity_map = { '"': '"', '&': '&', '<': '<', '>': '>', ''': "'", ' ': '\xa0', # Non-breaking space } for entity, char in entity_map.items(): text = text.replace(entity, char) return text def extract_math_equations(text: str) -> List[str]: """Extract multiple MathML equations from text starting with >> text = 'x and y' >>> equations = extract_math_equations(text) >>> len(equations) 2 """ # CRITICAL FIX: Do NOT decode HTML entities before regex extraction! # The input is XML/MathML which contains numeric character references # like < that are valid XML and must be preserved. # XML parsers (lxml) will handle entity decoding automatically. # Decoding them here would break valid XML like < → < # # Original line (REMOVED): text = decode_html_entities(text) equations = [] # Find all tags with their content # Use a more robust regex that handles nested tags and various formatting pattern = r']*>.*?' matches = re.findall(pattern, text, re.DOTALL | re.IGNORECASE) for match in matches: # Clean up the equation equation = match.strip() if equation: # Decode HTML entities in the extracted equation # This handles cases like " → " which are needed for valid XML equation = decode_html_entities(equation) equations.append(equation) # If no tags found, check if the entire input is a single equation if not equations and text.strip().startswith('' in text.lower(): # Decode HTML entities before appending text = decode_html_entities(text) equations.append(text) else: # Try to fix incomplete equation by adding closing tag if not text.lower().endswith(''): text += '' # Decode HTML entities before appending text = decode_html_entities(text) equations.append(text) return equations # Alias for backward compatibility (some code uses _local, some uses local_name) _local = local_name _text_content = text_content