Extrakt von Fortna gestartet um Standardsymbole und Basis für die ersten .rst Files zu erhalten.

2026-04-29 22:55:30 +02:00
parent 367eb38cc8
commit cfc494d0dd
20 changed files with 1092 additions and 3 deletions
@@ -0,0 +1,639 @@
+"""
+convert_to_rst.py - Konvertiert Word-Dokumente (.docx) und Word-erstellte PDFs nach .rst
+
+Zwei-Stufen-Ansatz:
+  1. Textextraktion: .docx via mammoth -> HTML, PDF via pdfminer -> Text
+  2. Strukturerkennung: HTML/Text -> .rst mit Ueberschriften-Hierarchie
+
+Verwendung:
+  python convert_to_rst.py <input_dir> <output_dir> [--dry-run] [--lang de]
+"""
+
+from __future__ import annotations
+
+import argparse
+import html as html_module
+import logging
+import os
+import re
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+# ---------------------------------------------------------------------------
+# Konfiguration: Mapping Quelldatei -> Zielkapitel
+# ---------------------------------------------------------------------------
+
+# Muster im Dateinamen -> Zielkapitel-Pfad (relativ zu chapters/)
+CHAPTER_MAPPING: list[tuple[re.Pattern[str], str]] = [
+    # 02_INSTRUCTIONS -> 04_betrieb, 05_wartung, 06_steuerung
+    (re.compile(r"betriebsanleitung", re.I), "04_betrieb/betriebsanleitung"),
+    (re.compile(r"wartungsanleitung|wartung", re.I), "05_wartung/wartungsanleitung"),
+    (re.compile(r"funktion.*(steuerung|control)|steuerungsbeschreibung", re.I), "06_steuerung"),
+    # 01_CERTIFICATES
+    (re.compile(r"konfigurationsbeschreibung|configuration.description", re.I), "01_zertifikate/konfigurationsbeschreibung"),
+    (re.compile(r"konformit|conformity|declaration", re.I), "01_zertifikate/konformitaetserklaerung"),
+    # 00_CONTENTS
+    (re.compile(r"contens|inhaltsverzeichnis|contents", re.I), "00_inhaltsverzeichnis"),
+    # 03_SPARE_PARTS -> 07_ersatzteile
+    (re.compile(r"spare.part|ersatzteil", re.I), "07_ersatzteile"),
+    # 05_REPORTS -> 10_anhang
+    (re.compile(r"wiederanlauf|drehrichtung|erprobung|protokoll|pr[uü]f", re.I), "10_anhang/pruefberichte"),
+    # Risikobewertung (RB_ prefix)
+    (re.compile(r"^RB_|risikobewertung|risikobeurteilung", re.I), "02_sicherheit/risikobewertung"),
+]
+
+# Benutzerdefinierte Word-Styles -> Heading-Level
+CUSTOM_STYLE_MAP = {
+    "Ü1-ILS": 1, "Ü2-ILS": 2, "Ü3-ILS": 3, "Ü4-ILS": 4,
+    "Ü1": 1, "Ü2": 2, "Ü3": 3, "Ü4": 4,
+}
+
+# mammoth style_map String
+MAMMOTH_STYLE_MAP = "\n".join([
+    "p[style-name='Ü1-ILS'] => h1:fresh",
+    "p[style-name='Ü2-ILS'] => h2:fresh",
+    "p[style-name='Ü3-ILS'] => h3:fresh",
+    "p[style-name='Ü4-ILS'] => h4:fresh",
+    "p[style-name='Heading 1'] => h1:fresh",
+    "p[style-name='Heading 2'] => h2:fresh",
+    "p[style-name='Heading 3'] => h3:fresh",
+    "p[style-name='Heading 4'] => h4:fresh",
+    "p[style-name='Doku-Text'] => p:fresh",
+    "p[style-name='Doku-1Aufzähl'] => ul > li:fresh",
+    "p[style-name='Doku-1Aufzählung'] => ul > li:fresh",
+    "p[style-name='Doku-2fettAufzählung'] => ul > li:fresh",
+    "p[style-name='Caption'] => p:fresh",
+    "p[style-name='Text'] => p:fresh",
+    "p[style-name='Doku-Text fett'] => p:fresh",
+    "p[style-name='Doku-Text12fett'] => p:fresh",
+    "p[style-name='Vorgabetext'] => p:fresh",
+])
+
+# rST Ueberschriften-Zeichen nach Level
+RST_HEADING_CHARS = {1: "=", 2: "-", 3: "~", 4: "^", 5: '"'}
+
+log = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Datenklassen
+# ---------------------------------------------------------------------------
+
+@dataclass
+class RstBlock:
+    """Ein Block im .rst-Dokument."""
+    kind: str  # "heading", "paragraph", "list_item", "image", "table", "empty"
+    text: str = ""
+    level: int = 0  # nur fuer headings
+    image_path: str = ""  # nur fuer images
+
+
+@dataclass
+class ConversionResult:
+    """Ergebnis einer Konvertierung."""
+    source_path: Path
+    target_chapter: str
+    rst_filename: str
+    blocks: list[RstBlock] = field(default_factory=list)
+    images: dict[str, bytes] = field(default_factory=dict)  # filename -> data
+    warnings: list[str] = field(default_factory=list)
+
+
+# ---------------------------------------------------------------------------
+# Kapitel-Erkennung
+# ---------------------------------------------------------------------------
+
+def detect_chapter(filename: str) -> str:
+    """Bestimmt das Zielkapitel anhand des Dateinamens."""
+    for pattern, chapter in CHAPTER_MAPPING:
+        if pattern.search(filename):
+            return chapter
+    return "10_anhang"  # Fallback
+
+
+def detect_steuerung_subdir(filename: str) -> str:
+    """Erkennt das Steuerungssystem fuer Unterordner in 06_steuerung/."""
+    lower = filename.lower()
+    if "loading" in lower or "loadingloop" in lower.replace("_", ""):
+        return "loading_loop"
+    if "omniflo" in lower:
+        return "omniflo"
+    if "ils" in lower:
+        return "ils"
+    return ""
+
+
+def make_rst_filename(source_name: str) -> str:
+    """Erzeugt einen sauberen .rst-Dateinamen."""
+    stem = Path(source_name).stem
+    # Versionsnummern und Datumsangaben vereinfachen
+    slug = stem.lower()
+    slug = re.sub(r"[^a-z0-9_]+", "_", slug)
+    slug = re.sub(r"_+", "_", slug).strip("_")
+    return slug + ".de.rst"
+
+
+# ---------------------------------------------------------------------------
+# Konvertierung: .docx -> HTML -> rST-Blocks
+# ---------------------------------------------------------------------------
+
+def convert_docx_to_blocks(docx_path: Path) -> tuple[list[RstBlock], dict[str, bytes], list[str]]:
+    """Konvertiert .docx via mammoth nach rST-Blocks."""
+    import mammoth
+
+    blocks: list[RstBlock] = []
+    images: dict[str, bytes] = {}
+    warnings: list[str] = []
+    image_counter = 0
+
+    def handle_image(image):
+        nonlocal image_counter
+        image_counter += 1
+        ext = image.content_type.split("/")[-1] if image.content_type else "png"
+        if ext == "jpeg":
+            ext = "jpg"
+        img_name = f"img_{image_counter:03d}.{ext}"
+        with image.open() as img_file:
+            images[img_name] = img_file.read()
+        return {"src": img_name}
+
+    with open(docx_path, "rb") as f:
+        result = mammoth.convert_to_html(
+            f,
+            style_map=MAMMOTH_STYLE_MAP,
+            convert_image=mammoth.images.img_element(handle_image),
+        )
+
+    html = result.value
+    for msg in result.messages:
+        warnings.append(str(msg))
+
+    # HTML parsen -> Blocks
+    blocks = _html_to_blocks(html)
+
+    return blocks, images, warnings
+
+
+def _html_to_blocks(html: str) -> list[RstBlock]:
+    """Parst mammoth-HTML in RstBlock-Liste."""
+    blocks: list[RstBlock] = []
+
+    # Einfacher Regex-basierter Parser (mammoth erzeugt sauberes HTML)
+    # Aufteilung in Top-Level-Elemente
+    elements = re.split(r"(</?(?:h[1-6]|p|ul|ol|li|table|tr|td|th|img)[^>]*>)", html)
+
+    current_text = ""
+    current_kind = "paragraph"
+    current_level = 0
+    in_list = False
+    in_table = False
+
+    for elem in elements:
+        if not elem:
+            continue
+
+        # Oeffnende Tags
+        heading_match = re.match(r"<h([1-6])(?:\s[^>]*)?>", elem)
+        if heading_match:
+            _flush_block(blocks, current_kind, current_text, current_level)
+            current_text = ""
+            current_kind = "heading"
+            current_level = int(heading_match.group(1))
+            continue
+
+        if re.match(r"</h[1-6]>", elem):
+            _flush_block(blocks, current_kind, current_text, current_level)
+            current_text = ""
+            current_kind = "paragraph"
+            current_level = 0
+            continue
+
+        if re.match(r"<ul[^>]*>", elem):
+            _flush_block(blocks, current_kind, current_text, current_level)
+            current_text = ""
+            in_list = True
+            continue
+
+        if re.match(r"</ul>", elem):
+            in_list = False
+            continue
+
+        if re.match(r"<li[^>]*>", elem):
+            _flush_block(blocks, current_kind, current_text, current_level)
+            current_text = ""
+            current_kind = "list_item"
+            continue
+
+        if re.match(r"</li>", elem):
+            _flush_block(blocks, current_kind, current_text, current_level)
+            current_text = ""
+            current_kind = "paragraph"
+            continue
+
+        if re.match(r"<p[^>]*>", elem):
+            _flush_block(blocks, current_kind, current_text, current_level)
+            current_text = ""
+            if current_kind != "heading":
+                current_kind = "paragraph"
+            continue
+
+        if re.match(r"</p>", elem):
+            _flush_block(blocks, current_kind, current_text, current_level)
+            current_text = ""
+            current_kind = "paragraph"
+            current_level = 0
+            continue
+
+        # Bilder
+        img_match = re.search(r'<img[^>]*src="([^"]+)"[^>]*/?\s*>', elem)
+        if img_match:
+            src = img_match.group(1)
+            if not src.startswith("data:"):  # Skip base64 inline images
+                blocks.append(RstBlock(kind="image", image_path=src))
+            continue
+
+        # table-Tags: vereinfacht als Text-Block
+        if re.match(r"<table[^>]*>", elem):
+            _flush_block(blocks, current_kind, current_text, current_level)
+            current_text = ""
+            in_table = True
+            continue
+        if re.match(r"</table>", elem):
+            in_table = False
+            _flush_block(blocks, "paragraph", current_text, 0)
+            current_text = ""
+            continue
+
+        # Text-Inhalt: HTML-Tags entfernen
+        text = re.sub(r"<[^>]+>", "", elem)
+        text = html_module.unescape(text)
+        current_text += text
+
+    _flush_block(blocks, current_kind, current_text, current_level)
+    return blocks
+
+
+def _flush_block(blocks: list[RstBlock], kind: str, text: str, level: int) -> None:
+    """Fuegt einen Block hinzu wenn Text vorhanden."""
+    clean = text.strip()
+    if not clean:
+        return
+    # Tab-getrennte Inhalte bereinigen
+    clean = re.sub(r"\t+", "  ", clean)
+    blocks.append(RstBlock(kind=kind, text=clean, level=level))
+
+
+# ---------------------------------------------------------------------------
+# Konvertierung: PDF -> Text -> rST-Blocks
+# ---------------------------------------------------------------------------
+
+def convert_pdf_to_blocks(pdf_path: Path) -> tuple[list[RstBlock], list[str]]:
+    """Konvertiert PDF via pdfminer nach rST-Blocks."""
+    from pdfminer.high_level import extract_text
+
+    warnings: list[str] = []
+
+    try:
+        text = extract_text(str(pdf_path))
+    except Exception as e:
+        warnings.append(f"PDF-Extraktion fehlgeschlagen: {e}")
+        return [], warnings
+
+    blocks = _text_to_blocks(text)
+    return blocks, warnings
+
+
+def _text_to_blocks(text: str) -> list[RstBlock]:
+    """Heuristik: Plain-Text in Blocks mit Ueberschriften-Erkennung."""
+    blocks: list[RstBlock] = []
+    lines = text.split("\n")
+
+    i = 0
+    while i < len(lines):
+        line = lines[i].rstrip()
+
+        if not line.strip():
+            i += 1
+            continue
+
+        # Heuristik: Nummerierte Ueberschriften (1, 1.1, 1.1.1 etc.)
+        heading_match = re.match(r"^(\d+(?:\.\d+)*)\s+(.+)$", line.strip())
+        if heading_match:
+            num = heading_match.group(1)
+            title = heading_match.group(2).strip()
+            level = min(num.count(".") + 1, 4)
+            # Pruefen ob die naechste Zeile leer ist (typisch fuer Ueberschriften)
+            next_empty = (i + 1 < len(lines) and not lines[i + 1].strip())
+            # Oder ob der Text kurz genug fuer eine Ueberschrift ist
+            if next_empty or len(title) < 80:
+                blocks.append(RstBlock(kind="heading", text=f"{num} {title}", level=level))
+                i += 1
+                continue
+
+        # Aufzaehlungszeichen
+        list_match = re.match(r"^\s*[•\-–]\s+(.+)$", line.strip())
+        if list_match:
+            blocks.append(RstBlock(kind="list_item", text=list_match.group(1).strip()))
+            i += 1
+            continue
+
+        # Normaler Absatz: Sammle zusammenhaengende Zeilen
+        para_lines = [line.strip()]
+        i += 1
+        while i < len(lines) and lines[i].strip() and not re.match(r"^\d+(?:\.\d+)*\s+", lines[i].strip()):
+            if re.match(r"^\s*[•\-–]\s+", lines[i].strip()):
+                break
+            para_lines.append(lines[i].strip())
+            i += 1
+
+        para_text = " ".join(para_lines)
+        if para_text:
+            blocks.append(RstBlock(kind="paragraph", text=para_text))
+
+    return blocks
+
+
+# ---------------------------------------------------------------------------
+# Blocks -> .rst Rendering
+# ---------------------------------------------------------------------------
+
+def render_rst(blocks: list[RstBlock], title: str = "") -> str:
+    """Rendert RstBlock-Liste als .rst-String."""
+    lines: list[str] = []
+
+    if title:
+        lines.append(RST_HEADING_CHARS[1] * len(title))
+        lines.append(title)
+        lines.append(RST_HEADING_CHARS[1] * len(title))
+        lines.append("")
+
+    prev_kind = ""
+    for block in blocks:
+        if block.kind == "heading":
+            text = block.text
+            if not text:
+                continue
+            level = max(1, min(block.level, 5))
+            char = RST_HEADING_CHARS[level]
+            lines.append("")
+            if level == 1:
+                lines.append(char * len(text))
+            lines.append(text)
+            lines.append(char * len(text))
+            lines.append("")
+
+        elif block.kind == "paragraph":
+            if prev_kind != "paragraph":
+                lines.append("")
+            lines.append(block.text)
+            lines.append("")
+
+        elif block.kind == "list_item":
+            lines.append(f"- {block.text}")
+
+        elif block.kind == "image":
+            lines.append("")
+            lines.append(f".. figure:: images/{block.image_path}")
+            lines.append("   :align: center")
+            lines.append("")
+
+        prev_kind = block.kind
+
+    # Bereinigung: Mehrfache Leerzeilen reduzieren
+    rst = "\n".join(lines)
+    rst = re.sub(r"\n{3,}", "\n\n", rst)
+    return rst.strip() + "\n"
+
+
+# ---------------------------------------------------------------------------
+# Erkennung: Ist PDF Word-erstellt?
+# ---------------------------------------------------------------------------
+
+def is_word_pdf(pdf_path: Path) -> bool:
+    """Prueft ob ein PDF mit Microsoft Word erstellt wurde."""
+    from pdfminer.pdfparser import PDFParser
+    from pdfminer.pdfdocument import PDFDocument
+
+    try:
+        with open(pdf_path, "rb") as f:
+            parser = PDFParser(f)
+            doc = PDFDocument(parser)
+            if not doc.info:
+                return False
+            info = doc.info[0]
+
+            def decode(v: object) -> str:
+                if isinstance(v, bytes):
+                    # UTF-16 BOM erkennen
+                    if v[:2] in (b"\xff\xfe", b"\xfe\xff"):
+                        return v.decode("utf-16", errors="replace")
+                    return v.decode("utf-8", errors="replace")
+                return str(v) if v else ""
+
+            creator = decode(info.get("Creator", b""))
+            producer = decode(info.get("Producer", b""))
+            combined = (creator + " " + producer).lower()
+            return "word" in combined or "microsoft" in combined
+    except Exception:
+        return False
+
+
+# ---------------------------------------------------------------------------
+# Hauptlogik
+# ---------------------------------------------------------------------------
+
+def find_sources(input_dir: Path, lang: str = "de") -> list[Path]:
+    """Findet alle konvertierbaren Quelldateien (DE)."""
+    sources: list[Path] = []
+
+    # 1. .docx am Root
+    for p in sorted(input_dir.glob("*.docx")):
+        sources.append(p)
+
+    # 2. Deutsche PDFs aus Unterordnern
+    lang_patterns = [f"*{lang}*", f"*{lang.upper()}*", f"*DE*", f"*_de_*"]
+    de_dirs = [f"*{lang}*", f"*{lang.upper()}*", "*DE*", "*original*"]
+
+    for pdf in sorted(input_dir.rglob("*.pdf")):
+        rel = pdf.relative_to(input_dir)
+        parts_lower = str(rel).lower()
+
+        # Nur DE-Dateien oder Dateien aus DE-Ordnern
+        is_de = False
+        for part in rel.parts:
+            pl = part.lower()
+            if "de" in pl.split("_") or pl.startswith("01_de") or pl.endswith("_de"):
+                is_de = True
+                break
+        if "_de." in parts_lower or "_de_" in parts_lower:
+            is_de = True
+
+        # Dateien im Root (ohne Sprachordner) sind auch DE
+        if len(rel.parts) <= 2:
+            is_de = True
+
+        if not is_de:
+            continue
+
+        # Nur Word-erstellte PDFs
+        if is_word_pdf(pdf):
+            sources.append(pdf)
+
+    return sources
+
+
+def convert_file(source: Path, input_dir: Path, chapters_dir: Path) -> ConversionResult:
+    """Konvertiert eine einzelne Datei und bestimmt Zielkapitel."""
+    filename = source.name
+
+    # Kapitel bestimmen
+    chapter = detect_chapter(filename)
+
+    # Spezialfall: Steuerung -> Unterordner
+    if chapter == "06_steuerung":
+        sub = detect_steuerung_subdir(filename)
+        if sub:
+            chapter = f"06_steuerung/{sub}"
+
+    rst_filename = make_rst_filename(filename)
+
+    result = ConversionResult(
+        source_path=source,
+        target_chapter=chapter,
+        rst_filename=rst_filename,
+    )
+
+    # Konvertierung
+    if source.suffix.lower() == ".docx":
+        blocks, images, warnings = convert_docx_to_blocks(source)
+        result.blocks = blocks
+        result.images = images
+        result.warnings = warnings
+    elif source.suffix.lower() == ".pdf":
+        blocks, warnings = convert_pdf_to_blocks(source)
+        result.blocks = blocks
+        result.warnings = warnings
+    else:
+        result.warnings.append(f"Unbekanntes Format: {source.suffix}")
+
+    return result
+
+
+def write_result(result: ConversionResult, chapters_dir: Path, dry_run: bool = False) -> None:
+    """Schreibt das Konvertierungsergebnis ins Dateisystem."""
+    target_dir = chapters_dir / result.target_chapter
+    target_file = target_dir / result.rst_filename
+
+    # Titel aus erstem Heading oder Dateiname
+    title = ""
+    for block in result.blocks:
+        if block.kind == "heading" and block.text:
+            title = block.text
+            break
+    if not title:
+        title = result.source_path.stem
+
+    rst_content = render_rst(result.blocks, title="")
+
+    if dry_run:
+        log.info("[DRY-RUN] %s -> %s (%d blocks, %d images)",
+                 result.source_path.name, target_file.relative_to(chapters_dir),
+                 len(result.blocks), len(result.images))
+        return
+
+    # Ordner anlegen
+    target_dir.mkdir(parents=True, exist_ok=True)
+
+    # .rst schreiben
+    target_file.write_text(rst_content, encoding="utf-8")
+    log.info("Geschrieben: %s (%d blocks)", target_file.relative_to(chapters_dir), len(result.blocks))
+
+    # Bilder speichern
+    if result.images:
+        img_dir = target_dir / "images"
+        img_dir.mkdir(exist_ok=True)
+        for img_name, img_data in result.images.items():
+            img_file = img_dir / img_name
+            img_file.write_bytes(img_data)
+        log.info("  %d Bilder nach %s", len(result.images), img_dir.relative_to(chapters_dir))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Konvertiert Word-Dokumente (.docx) und Word-PDFs nach .rst"
+    )
+    parser.add_argument("input_dir", type=Path,
+                        help="Quellverzeichnis mit .docx/.pdf Dateien")
+    parser.add_argument("output_dir", type=Path,
+                        help="Ziel-chapters/ Verzeichnis")
+    parser.add_argument("--dry-run", action="store_true",
+                        help="Nur anzeigen, nichts schreiben")
+    parser.add_argument("--lang", default="de",
+                        help="Zielsprache (default: de)")
+    parser.add_argument("--verbose", "-v", action="store_true",
+                        help="Ausfuehrliche Ausgabe")
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(levelname)-7s %(message)s",
+    )
+
+    input_dir = args.input_dir.resolve()
+    chapters_dir = args.output_dir.resolve()
+
+    if not input_dir.exists():
+        log.error("Quellverzeichnis nicht gefunden: %s", input_dir)
+        sys.exit(1)
+
+    if not chapters_dir.exists():
+        log.error("Zielverzeichnis nicht gefunden: %s", chapters_dir)
+        sys.exit(1)
+
+    # Quellen finden
+    log.info("Suche Quelldateien in %s ...", input_dir)
+    sources = find_sources(input_dir, args.lang)
+    log.info("Gefunden: %d Dateien", len(sources))
+
+    if not sources:
+        log.warning("Keine konvertierbaren Dateien gefunden.")
+        sys.exit(0)
+
+    # Anzeige
+    for src in sources:
+        rel = src.relative_to(input_dir)
+        chapter = detect_chapter(src.name)
+        if chapter == "06_steuerung":
+            sub = detect_steuerung_subdir(src.name)
+            if sub:
+                chapter = f"06_steuerung/{sub}"
+        log.info("  %s -> %s", rel, chapter)
+
+    print()
+
+    # Konvertierung
+    success = 0
+    errors = 0
+    for src in sources:
+        try:
+            result = convert_file(src, input_dir, chapters_dir)
+            if result.blocks:
+                write_result(result, chapters_dir, dry_run=args.dry_run)
+                success += 1
+            else:
+                log.warning("Keine Inhalte extrahiert: %s", src.name)
+                errors += 1
+            for w in result.warnings:
+                log.warning("  %s: %s", src.name, w)
+        except Exception as e:
+            log.error("Fehler bei %s: %s", src.name, e)
+            errors += 1
+
+    print()
+    log.info("Fertig: %d erfolgreich, %d Fehler", success, errors)
+
+
+if __name__ == "__main__":
+    main()