#!/usr/bin/env python

"""

pybind11 documentation insertion tool.

Extracts documentation from Doxygen XML and inserts it into pybind11 bindings

using string matching via signature lookup table.

Logic:

- Builds a multi-key lookup for all functions (MaterialX::, mx::, Class::method, method)

- Handles free functions without <qualifiedname> by assuming MaterialX namespace

- Adds class context tracking to correctly document lambda-based bindings

- Supports .def(...) and .def_static(...); skips .def_readonly_static(...)

"""

import argparse

import re

import json

import xml.etree.ElementTree as ET

from pathlib import Path

from typing import Dict, Optional

# Defaults (can be overridden by CLI)

DOXYGEN_XML_DIR = Path("build/documents/doxygen_xml")

PYBIND_DIR = Path("source/PyMaterialX")

class DocExtractor:

"""Extracts documentation from Doxygen XML files and builds a lookup table."""

def __init__(self, xml_dir: Path):

self.xml_dir = xml_dir

self.class_docs: Dict[str, str] = {}

self.func_docs: Dict[str, Dict] = {}

# Multi-key lookup: all name variants point to the same doc

self.func_lookup: Dict[str, Dict] = {}

def extract(self):

if not self.xml_dir.exists():

raise FileNotFoundError(f"Doxygen XML directory not found: {self.xml_dir}")

for xml_file in self.xml_dir.glob("*.xml"):

self._process_xml_file(xml_file)

self._build_lookup_table()

print(f"Extracted {len(self.class_docs)} classes and {len(self.func_docs)} functions")

print(f"Built lookup table with {len(self.func_lookup)} keys")

def _process_xml_file(self, xml_file: Path):

tree = ET.parse(xml_file)

root = tree.getroot()

# Class / struct documentation

for compound in root.findall(".//compounddef[@kind='class']") + root.findall(".//compounddef[@kind='struct']"):

self._extract_class_doc(compound)

# Function documentation

for member in root.findall(".//memberdef[@kind='function']"):

self._extract_func_doc(member)

def _extract_class_doc(self, compound):

name = self._get_text(compound.find("compoundname"))

brief = self._get_text(compound.find("briefdescription/para"))

detail = self._extract_detail(compound.find("detaileddescription"))

doc = "\n\n".join(filter(None, [brief, detail]))

if doc:

normalized = self._normalize_name(name)

self.class_docs[normalized] = doc

def _extract_func_doc(self, member):

name = self._get_text(member.find("name"))

qualified = self._get_text(member.find("qualifiedname"))

# Many free functions have no <qualifiedname>; use the bare name

# and normalize to MaterialX::name so lookups can resolve.

if not qualified and name:

qualified = name

if not qualified:

return

brief = self._get_text(member.find("briefdescription/para"))

detail = self._extract_detail(member.find("detaileddescription"))

params = self._extract_params(member)

returns = self._get_text(member.find(".//simplesect[@kind='return']"))

normalized = self._normalize_name(qualified)

self.func_docs[normalized] = {

"brief": brief,

"detail": detail,

"params": params,

"returns": returns,

}

def _build_lookup_table(self):

for qualified_name, doc in self.func_docs.items():

for variant in self._generate_name_variants(qualified_name):

if variant not in self.func_lookup:

self.func_lookup[variant] = doc

def _generate_name_variants(self, qualified_name: str) -> list:

variants = [qualified_name]

parts = qualified_name.split("::")

# Class::method

if len(parts) >= 2:

variants.append("::".join(parts[-2:]))

# method

if len(parts) >= 1:

variants.append(parts[-1])

# mx:: variant if MaterialX::

if qualified_name.startswith("MaterialX::"):

mx_variant = qualified_name.replace("MaterialX::", "mx::", 1)

variants.append(mx_variant)

if len(parts) >= 3:

variants.append(f"mx::{parts[-2]}::{parts[-1]}")

return variants

def _normalize_name(self, name: str) -> str:

if not name:

return name

return name if name.startswith("MaterialX::") else f"MaterialX::{name}"

def _get_text(self, elem) -> str:

if elem is None:

return ""

text = "".join(elem.itertext())

return re.sub(r"\s+", " ", text).strip()

def _extract_detail(self, elem, exclude_tags={"parameterlist", "simplesect"}) -> str:

if elem is None:

return ""

parts = []

for para in elem.findall("para"):

if not any(para.find(tag) is not None for tag in exclude_tags):

t = self._get_text(para)

if t:

parts.append(t)

return "\n\n".join(parts)

def _extract_params(self, member) -> Dict[str, str]:

params = {}

for param_item in member.findall(".//parameterlist[@kind='param']/parameteritem"):

name = self._get_text(param_item.find("parameternamelist/parametername"))

desc = self._get_text(param_item.find("parameterdescription"))

if name:

params[name] = desc

return params

class DocInserter:

"""Inserts documentation into pybind11 binding files."""

def __init__(self, extractor: DocExtractor, pybind_dir: Path, force_replace: bool = False):

self.extractor = extractor

self.pybind_dir = pybind_dir

self.force_replace = force_replace

self.class_pattern = re.compile(r"py::class_<")

self.def_pattern = re.compile(r"\.def(?:_static)?\s*\(")

# Match .def and .def_static; skip .def_readonly_static (constants)

self.def_pattern = re.compile(r"\.def(?:_static)?\s*\(")

self.skip_pattern = re.compile(r"\.def_readonly_static\s*\(")

def process_all_files(self):

cpp_files = list(self.pybind_dir.rglob("*.cpp"))

patched = 0

for cpp_file in cpp_files:

if self._process_file(cpp_file):

patched += 1

print(f"\nProcessed {len(cpp_files)} files, patched {patched}")

def _process_file(self, cpp_file: Path) -> bool:

content = cpp_file.read_text(encoding="utf-8")

original = content

content = self._insert_class_docs(content)

content = self._insert_method_docs(content)

if content != original:

cpp_file.write_text(content, encoding="utf-8")

print(f" - {cpp_file.relative_to(self.pybind_dir.parent)}")

return True

else:

print(f" - {cpp_file.relative_to(self.pybind_dir.parent)}")

return False

def _insert_class_docs(self, content: str) -> str:

result = []

pos = 0

for match in self.class_pattern.finditer(content):

result.append(content[pos:match.start()])

start = match.start()

template_end = self._find_template_end(content, start)

if template_end == -1:

result.append(content[start:match.end()])

pos = match.end()

continue

paren_start = content.find('(', template_end)

if paren_start == -1:

result.append(content[start:match.end()])

pos = match.end()

continue

paren_end = self._find_matching_paren(content, paren_start)

if paren_end == -1:

result.append(content[start:match.end()])

pos = match.end()

continue

args_text = content[paren_start + 1:paren_end]

class_name = self._extract_class_name(args_text)

if class_name:

doc = self.extractor.class_docs.get(self.extractor._normalize_name(class_name))

if doc:

args = self._split_args(args_text)

if len(args) >= 3 and not self.force_replace:

result.append(content[start:paren_end + 1])

pos = paren_end + 1

continue

escaped = self._escape_for_cpp(doc)

if len(args) >= 3 and self.force_replace:

new_args = args[:2] + [f'"{escaped}"'] + args[3:]

result.append(content[start:paren_start + 1])

result.append(", ".join(new_args))

result.append(")")

else:

result.append(content[start:paren_end])

result.append(f', "{escaped}")')

pos = paren_end + 1

continue

result.append(content[start:paren_end + 1])

pos = paren_end + 1

result.append(content[pos:])

return "".join(result)

def _insert_method_docs(self, content: str) -> str:

# Build a map of line numbers to class contexts

class_contexts = self._extract_class_contexts(content)

result = []

pos = 0

for match in self.def_pattern.finditer(content):

if self.skip_pattern.match(content, match.start()):

continue

result.append(content[pos:match.start()])

start = match.start()

paren_start = content.find('(', start)

if paren_start == -1:

result.append(content[start:match.end()])

pos = match.end()

continue

paren_end = self._find_matching_paren(content, paren_start)

if paren_end == -1:

result.append(content[start:match.end()])

pos = match.end()

continue

args_text = content[paren_start + 1:paren_end]

args = self._split_args(args_text)

if len(args) < 2:

result.append(content[start:paren_end + 1])

pos = paren_end + 1

continue

has_doc = self._has_docstring(args)

if has_doc and not self.force_replace:

result.append(content[start:paren_end + 1])

pos = paren_end + 1

continue

callable_ref = args[1].strip()

current_line = content[:start].count('\n')

class_context = class_contexts.get(current_line)

doc_entry = self._find_doc_for_callable(callable_ref, class_context)

if doc_entry:

docstring = self._build_docstring(doc_entry)

escaped = self._escape_for_cpp(docstring)

if has_doc and self.force_replace:

doc_idx = self._find_docstring_arg_index(args)

if doc_idx is not None:

new_args = args[:doc_idx] + [f'"{escaped}"'] + args[doc_idx + 1:]

result.append(content[start:paren_start + 1])

result.append(", ".join(new_args))

result.append(")")

pos = paren_end + 1

continue

result.append(content[start:paren_end])

result.append(f', "{escaped}")')

pos = paren_end + 1

continue

result.append(content[start:paren_end + 1])

pos = paren_end + 1

result.append(content[pos:])

return "".join(result)

def _extract_class_contexts(self, content: str) -> Dict[int, str]:

contexts = {}

for match in self.class_pattern.finditer(content):

start = match.start()

template_end = self._find_template_end(content, start)

if template_end == -1:

continue

template_start = content.find('<', start) + 1

template_content = content[template_start:template_end - 1]

class_type = template_content.split(',')[0].strip()

class_name = class_type.split('::')[-1] if '::' in class_type else class_type

start_line = content[:start].count('\n')

end_pos = content.find(';', start)

if end_pos != -1:

end_line = content[:end_pos].count('\n')

for line in range(start_line, end_line + 1):

contexts[line] = class_name

return contexts

def _find_doc_for_callable(self, callable_ref: str, class_context: Optional[str] = None) -> Optional[Dict]:

callable_ref = callable_ref.strip()

# Function pointers like &mx::Class::method or &MaterialX::name

if callable_ref.startswith('&'):

name = callable_ref[1:].strip()

name = re.sub(r'[,\s]+$', '', name)

return self.extractor.func_lookup.get(name)

# Lambdas: look for elem.method( or obj->method(

method_match = re.search(r'[\.\->](\w+)\s*\(', callable_ref)

if method_match:

method_name = method_match.group(1)

if class_context:

for prefix in ("", "mx::", "MaterialX::"):

qualified = f"{prefix}{class_context}::{method_name}" if prefix else f"{class_context}::{method_name}"

doc = self.extractor.func_lookup.get(qualified)

if doc:

return doc

return self.extractor.func_lookup.get(method_name)

return None

def _build_docstring(self, doc_entry: Dict) -> str:

parts = []

if doc_entry.get("brief"):

parts.append(doc_entry["brief"])

if doc_entry.get("detail"):

parts.append(doc_entry["detail"])

params = doc_entry.get("params", {})

if params:

param_lines = ["Args:"]

for name, desc in params.items():

param_lines.append(f" {name}: {desc}" if desc else f" {name}:")

parts.append("\n".join(param_lines))

if doc_entry.get("returns"):

parts.append(f"Returns:\n {doc_entry['returns']}")

return "\n\n".join(parts)

def _escape_for_cpp(self, s: str) -> str:

if not s:

return ""

s = s.replace("\\", "\\\\").replace('"', '\\"')

s = s.replace("\n", "\\n")

return s

def _find_template_end(self, content: str, start: int) -> int:

pos = content.find('<', start)

if pos == -1:

return -1

depth = 1

i = pos + 1

in_string = False

while i < len(content) and depth > 0:

c = content[i]

if c == '"' and content[i - 1] != '\\':

in_string = not in_string

elif not in_string:

if c == '<':

depth += 1

elif c == '>':

depth -= 1

i += 1

return i if depth == 0 else -1

def _find_matching_paren(self, content: str, start: int) -> int:

depth = 0

in_string = False

escape = False

for i in range(start, len(content)):

c = content[i]

if escape:

escape = False

continue

if c == '\\':

escape = True

continue

if c == '"':

in_string = not in_string

continue

if not in_string:

if c == '(':

depth += 1

elif c == ')':

depth -= 1

if depth == 0:

return i

return -1

def _split_args(self, args_text: str) -> list:

args = []

current = []

depth = 0

in_string = False

escape = False

for c in args_text:

if escape:

current.append(c)

escape = False

continue

if c == '\\':

current.append(c)

escape = True

continue

if c == '"':

in_string = not in_string

current.append(c)

continue

if not in_string:

if c in '(<':

depth += 1

elif c in ')>':

depth -= 1

elif c == ',' and depth == 0:

args.append("".join(current).strip())

current = []

continue

current.append(c)

if current:

args.append("".join(current).strip())

return args

def _extract_class_name(self, args_text: str) -> Optional[str]:

args = self._split_args(args_text)

if len(args) >= 2:

return args[1].strip().strip('"')

return None

def _has_docstring(self, args: list) -> bool:

for arg in args[2:]:

a = arg.strip()

if not a.startswith("py::arg") and a.startswith('"'):

return True

return False

def _find_docstring_arg_index(self, args: list) -> Optional[int]:

for i, arg in enumerate(args[2:], start=2):

a = arg.strip()

if not a.startswith("py::arg") and a.startswith('"'):

return i

return None

def main():

parser = argparse.ArgumentParser(description="Extract Doxygen docs and insert into pybind11 bindings (simplified)")

parser.add_argument("-d", "--doxygen_xml_dir", type=Path, default=Path("build/documents/doxygen_xml"), help="Path to Doxygen XML output directory")

parser.add_argument("-p", "--pybind_dir", type=Path, default=Path("source/PyMaterialX"), help="Path to pybind11 bindings directory")

parser.add_argument("-f", "--force", action="store_true", help="Force replace existing docstrings")

parser.add_argument("-j", "--write_json", action="store_true", help="Write extracted docs to JSON files")

args = parser.parse_args()

if not args.doxygen_xml_dir.exists():

print(f"Error: Doxygen XML directory not found: {args.doxygen_xml_dir}")

return 1

if not args.pybind_dir.exists():

print(f"Error: Pybind directory not found: {args.pybind_dir}")

return 1

print("Extracting documentation from Doxygen XML...")

extractor = DocExtractor(args.doxygen_xml_dir)

extractor.extract()

if args.write_json:

print("\nWriting JSON files...")

Path("class_docs.json").write_text(json.dumps(extractor.class_docs, indent=2), encoding="utf-8")

Path("func_docs.json").write_text(json.dumps(extractor.func_docs, indent=2), encoding="utf-8")

print(" - class_docs.json")

print(" - func_docs.json")

print(f"\n{'Replacing' if args.force else 'Inserting'} documentation in pybind11 files...")

inserter = DocInserter(extractor, args.pybind_dir, args.force)

inserter.process_all_files()

print("\nDone!")

return 0

if __name__ == "__main__":

exit(main())