Files
doc_build/lib/convert_to_rst.py

640 lines
21 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
convert_to_rst.py - Konvertiert Word-Dokumente (.docx) und Word-erstellte PDFs nach .rst
Zwei-Stufen-Ansatz:
1. Textextraktion: .docx via mammoth -> HTML, PDF via pdfminer -> Text
2. Strukturerkennung: HTML/Text -> .rst mit Ueberschriften-Hierarchie
Verwendung:
python convert_to_rst.py <input_dir> <output_dir> [--dry-run] [--lang de]
"""
from __future__ import annotations
import argparse
import html as html_module
import logging
import os
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional
# ---------------------------------------------------------------------------
# Konfiguration: Mapping Quelldatei -> Zielkapitel
# ---------------------------------------------------------------------------
# Muster im Dateinamen -> Zielkapitel-Pfad (relativ zu chapters/)
CHAPTER_MAPPING: list[tuple[re.Pattern[str], str]] = [
# 02_INSTRUCTIONS -> 04_betrieb, 05_wartung, 06_steuerung
(re.compile(r"betriebsanleitung", re.I), "04_betrieb/betriebsanleitung"),
(re.compile(r"wartungsanleitung|wartung", re.I), "05_wartung/wartungsanleitung"),
(re.compile(r"funktion.*(steuerung|control)|steuerungsbeschreibung", re.I), "06_steuerung"),
# 01_CERTIFICATES
(re.compile(r"konfigurationsbeschreibung|configuration.description", re.I), "01_zertifikate/konfigurationsbeschreibung"),
(re.compile(r"konformit|conformity|declaration", re.I), "01_zertifikate/konformitaetserklaerung"),
# 00_CONTENTS
(re.compile(r"contens|inhaltsverzeichnis|contents", re.I), "00_inhaltsverzeichnis"),
# 03_SPARE_PARTS -> 07_ersatzteile
(re.compile(r"spare.part|ersatzteil", re.I), "07_ersatzteile"),
# 05_REPORTS -> 10_anhang
(re.compile(r"wiederanlauf|drehrichtung|erprobung|protokoll|pr[uü]f", re.I), "10_anhang/pruefberichte"),
# Risikobewertung (RB_ prefix)
(re.compile(r"^RB_|risikobewertung|risikobeurteilung", re.I), "02_sicherheit/risikobewertung"),
]
# Benutzerdefinierte Word-Styles -> Heading-Level
CUSTOM_STYLE_MAP = {
"Ü1-ILS": 1, "Ü2-ILS": 2, "Ü3-ILS": 3, "Ü4-ILS": 4,
"Ü1": 1, "Ü2": 2, "Ü3": 3, "Ü4": 4,
}
# mammoth style_map String
MAMMOTH_STYLE_MAP = "\n".join([
"p[style-name='Ü1-ILS'] => h1:fresh",
"p[style-name='Ü2-ILS'] => h2:fresh",
"p[style-name='Ü3-ILS'] => h3:fresh",
"p[style-name='Ü4-ILS'] => h4:fresh",
"p[style-name='Heading 1'] => h1:fresh",
"p[style-name='Heading 2'] => h2:fresh",
"p[style-name='Heading 3'] => h3:fresh",
"p[style-name='Heading 4'] => h4:fresh",
"p[style-name='Doku-Text'] => p:fresh",
"p[style-name='Doku-1Aufzähl'] => ul > li:fresh",
"p[style-name='Doku-1Aufzählung'] => ul > li:fresh",
"p[style-name='Doku-2fettAufzählung'] => ul > li:fresh",
"p[style-name='Caption'] => p:fresh",
"p[style-name='Text'] => p:fresh",
"p[style-name='Doku-Text fett'] => p:fresh",
"p[style-name='Doku-Text12fett'] => p:fresh",
"p[style-name='Vorgabetext'] => p:fresh",
])
# rST Ueberschriften-Zeichen nach Level
RST_HEADING_CHARS = {1: "=", 2: "-", 3: "~", 4: "^", 5: '"'}
log = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Datenklassen
# ---------------------------------------------------------------------------
@dataclass
class RstBlock:
"""Ein Block im .rst-Dokument."""
kind: str # "heading", "paragraph", "list_item", "image", "table", "empty"
text: str = ""
level: int = 0 # nur fuer headings
image_path: str = "" # nur fuer images
@dataclass
class ConversionResult:
"""Ergebnis einer Konvertierung."""
source_path: Path
target_chapter: str
rst_filename: str
blocks: list[RstBlock] = field(default_factory=list)
images: dict[str, bytes] = field(default_factory=dict) # filename -> data
warnings: list[str] = field(default_factory=list)
# ---------------------------------------------------------------------------
# Kapitel-Erkennung
# ---------------------------------------------------------------------------
def detect_chapter(filename: str) -> str:
"""Bestimmt das Zielkapitel anhand des Dateinamens."""
for pattern, chapter in CHAPTER_MAPPING:
if pattern.search(filename):
return chapter
return "10_anhang" # Fallback
def detect_steuerung_subdir(filename: str) -> str:
"""Erkennt das Steuerungssystem fuer Unterordner in 06_steuerung/."""
lower = filename.lower()
if "loading" in lower or "loadingloop" in lower.replace("_", ""):
return "loading_loop"
if "omniflo" in lower:
return "omniflo"
if "ils" in lower:
return "ils"
return ""
def make_rst_filename(source_name: str) -> str:
"""Erzeugt einen sauberen .rst-Dateinamen."""
stem = Path(source_name).stem
# Versionsnummern und Datumsangaben vereinfachen
slug = stem.lower()
slug = re.sub(r"[^a-z0-9_]+", "_", slug)
slug = re.sub(r"_+", "_", slug).strip("_")
return slug + ".de.rst"
# ---------------------------------------------------------------------------
# Konvertierung: .docx -> HTML -> rST-Blocks
# ---------------------------------------------------------------------------
def convert_docx_to_blocks(docx_path: Path) -> tuple[list[RstBlock], dict[str, bytes], list[str]]:
"""Konvertiert .docx via mammoth nach rST-Blocks."""
import mammoth
blocks: list[RstBlock] = []
images: dict[str, bytes] = {}
warnings: list[str] = []
image_counter = 0
def handle_image(image):
nonlocal image_counter
image_counter += 1
ext = image.content_type.split("/")[-1] if image.content_type else "png"
if ext == "jpeg":
ext = "jpg"
img_name = f"img_{image_counter:03d}.{ext}"
with image.open() as img_file:
images[img_name] = img_file.read()
return {"src": img_name}
with open(docx_path, "rb") as f:
result = mammoth.convert_to_html(
f,
style_map=MAMMOTH_STYLE_MAP,
convert_image=mammoth.images.img_element(handle_image),
)
html = result.value
for msg in result.messages:
warnings.append(str(msg))
# HTML parsen -> Blocks
blocks = _html_to_blocks(html)
return blocks, images, warnings
def _html_to_blocks(html: str) -> list[RstBlock]:
"""Parst mammoth-HTML in RstBlock-Liste."""
blocks: list[RstBlock] = []
# Einfacher Regex-basierter Parser (mammoth erzeugt sauberes HTML)
# Aufteilung in Top-Level-Elemente
elements = re.split(r"(</?(?:h[1-6]|p|ul|ol|li|table|tr|td|th|img)[^>]*>)", html)
current_text = ""
current_kind = "paragraph"
current_level = 0
in_list = False
in_table = False
for elem in elements:
if not elem:
continue
# Oeffnende Tags
heading_match = re.match(r"<h([1-6])(?:\s[^>]*)?>", elem)
if heading_match:
_flush_block(blocks, current_kind, current_text, current_level)
current_text = ""
current_kind = "heading"
current_level = int(heading_match.group(1))
continue
if re.match(r"</h[1-6]>", elem):
_flush_block(blocks, current_kind, current_text, current_level)
current_text = ""
current_kind = "paragraph"
current_level = 0
continue
if re.match(r"<ul[^>]*>", elem):
_flush_block(blocks, current_kind, current_text, current_level)
current_text = ""
in_list = True
continue
if re.match(r"</ul>", elem):
in_list = False
continue
if re.match(r"<li[^>]*>", elem):
_flush_block(blocks, current_kind, current_text, current_level)
current_text = ""
current_kind = "list_item"
continue
if re.match(r"</li>", elem):
_flush_block(blocks, current_kind, current_text, current_level)
current_text = ""
current_kind = "paragraph"
continue
if re.match(r"<p[^>]*>", elem):
_flush_block(blocks, current_kind, current_text, current_level)
current_text = ""
if current_kind != "heading":
current_kind = "paragraph"
continue
if re.match(r"</p>", elem):
_flush_block(blocks, current_kind, current_text, current_level)
current_text = ""
current_kind = "paragraph"
current_level = 0
continue
# Bilder
img_match = re.search(r'<img[^>]*src="([^"]+)"[^>]*/?\s*>', elem)
if img_match:
src = img_match.group(1)
if not src.startswith("data:"): # Skip base64 inline images
blocks.append(RstBlock(kind="image", image_path=src))
continue
# table-Tags: vereinfacht als Text-Block
if re.match(r"<table[^>]*>", elem):
_flush_block(blocks, current_kind, current_text, current_level)
current_text = ""
in_table = True
continue
if re.match(r"</table>", elem):
in_table = False
_flush_block(blocks, "paragraph", current_text, 0)
current_text = ""
continue
# Text-Inhalt: HTML-Tags entfernen
text = re.sub(r"<[^>]+>", "", elem)
text = html_module.unescape(text)
current_text += text
_flush_block(blocks, current_kind, current_text, current_level)
return blocks
def _flush_block(blocks: list[RstBlock], kind: str, text: str, level: int) -> None:
"""Fuegt einen Block hinzu wenn Text vorhanden."""
clean = text.strip()
if not clean:
return
# Tab-getrennte Inhalte bereinigen
clean = re.sub(r"\t+", " ", clean)
blocks.append(RstBlock(kind=kind, text=clean, level=level))
# ---------------------------------------------------------------------------
# Konvertierung: PDF -> Text -> rST-Blocks
# ---------------------------------------------------------------------------
def convert_pdf_to_blocks(pdf_path: Path) -> tuple[list[RstBlock], list[str]]:
"""Konvertiert PDF via pdfminer nach rST-Blocks."""
from pdfminer.high_level import extract_text
warnings: list[str] = []
try:
text = extract_text(str(pdf_path))
except Exception as e:
warnings.append(f"PDF-Extraktion fehlgeschlagen: {e}")
return [], warnings
blocks = _text_to_blocks(text)
return blocks, warnings
def _text_to_blocks(text: str) -> list[RstBlock]:
"""Heuristik: Plain-Text in Blocks mit Ueberschriften-Erkennung."""
blocks: list[RstBlock] = []
lines = text.split("\n")
i = 0
while i < len(lines):
line = lines[i].rstrip()
if not line.strip():
i += 1
continue
# Heuristik: Nummerierte Ueberschriften (1, 1.1, 1.1.1 etc.)
heading_match = re.match(r"^(\d+(?:\.\d+)*)\s+(.+)$", line.strip())
if heading_match:
num = heading_match.group(1)
title = heading_match.group(2).strip()
level = min(num.count(".") + 1, 4)
# Pruefen ob die naechste Zeile leer ist (typisch fuer Ueberschriften)
next_empty = (i + 1 < len(lines) and not lines[i + 1].strip())
# Oder ob der Text kurz genug fuer eine Ueberschrift ist
if next_empty or len(title) < 80:
blocks.append(RstBlock(kind="heading", text=f"{num} {title}", level=level))
i += 1
continue
# Aufzaehlungszeichen
list_match = re.match(r"^\s*[•\-]\s+(.+)$", line.strip())
if list_match:
blocks.append(RstBlock(kind="list_item", text=list_match.group(1).strip()))
i += 1
continue
# Normaler Absatz: Sammle zusammenhaengende Zeilen
para_lines = [line.strip()]
i += 1
while i < len(lines) and lines[i].strip() and not re.match(r"^\d+(?:\.\d+)*\s+", lines[i].strip()):
if re.match(r"^\s*[•\-]\s+", lines[i].strip()):
break
para_lines.append(lines[i].strip())
i += 1
para_text = " ".join(para_lines)
if para_text:
blocks.append(RstBlock(kind="paragraph", text=para_text))
return blocks
# ---------------------------------------------------------------------------
# Blocks -> .rst Rendering
# ---------------------------------------------------------------------------
def render_rst(blocks: list[RstBlock], title: str = "") -> str:
"""Rendert RstBlock-Liste als .rst-String."""
lines: list[str] = []
if title:
lines.append(RST_HEADING_CHARS[1] * len(title))
lines.append(title)
lines.append(RST_HEADING_CHARS[1] * len(title))
lines.append("")
prev_kind = ""
for block in blocks:
if block.kind == "heading":
text = block.text
if not text:
continue
level = max(1, min(block.level, 5))
char = RST_HEADING_CHARS[level]
lines.append("")
if level == 1:
lines.append(char * len(text))
lines.append(text)
lines.append(char * len(text))
lines.append("")
elif block.kind == "paragraph":
if prev_kind != "paragraph":
lines.append("")
lines.append(block.text)
lines.append("")
elif block.kind == "list_item":
lines.append(f"- {block.text}")
elif block.kind == "image":
lines.append("")
lines.append(f".. figure:: images/{block.image_path}")
lines.append(" :align: center")
lines.append("")
prev_kind = block.kind
# Bereinigung: Mehrfache Leerzeilen reduzieren
rst = "\n".join(lines)
rst = re.sub(r"\n{3,}", "\n\n", rst)
return rst.strip() + "\n"
# ---------------------------------------------------------------------------
# Erkennung: Ist PDF Word-erstellt?
# ---------------------------------------------------------------------------
def is_word_pdf(pdf_path: Path) -> bool:
"""Prueft ob ein PDF mit Microsoft Word erstellt wurde."""
from pdfminer.pdfparser import PDFParser
from pdfminer.pdfdocument import PDFDocument
try:
with open(pdf_path, "rb") as f:
parser = PDFParser(f)
doc = PDFDocument(parser)
if not doc.info:
return False
info = doc.info[0]
def decode(v: object) -> str:
if isinstance(v, bytes):
# UTF-16 BOM erkennen
if v[:2] in (b"\xff\xfe", b"\xfe\xff"):
return v.decode("utf-16", errors="replace")
return v.decode("utf-8", errors="replace")
return str(v) if v else ""
creator = decode(info.get("Creator", b""))
producer = decode(info.get("Producer", b""))
combined = (creator + " " + producer).lower()
return "word" in combined or "microsoft" in combined
except Exception:
return False
# ---------------------------------------------------------------------------
# Hauptlogik
# ---------------------------------------------------------------------------
def find_sources(input_dir: Path, lang: str = "de") -> list[Path]:
"""Findet alle konvertierbaren Quelldateien (DE)."""
sources: list[Path] = []
# 1. .docx am Root
for p in sorted(input_dir.glob("*.docx")):
sources.append(p)
# 2. Deutsche PDFs aus Unterordnern
lang_patterns = [f"*{lang}*", f"*{lang.upper()}*", f"*DE*", f"*_de_*"]
de_dirs = [f"*{lang}*", f"*{lang.upper()}*", "*DE*", "*original*"]
for pdf in sorted(input_dir.rglob("*.pdf")):
rel = pdf.relative_to(input_dir)
parts_lower = str(rel).lower()
# Nur DE-Dateien oder Dateien aus DE-Ordnern
is_de = False
for part in rel.parts:
pl = part.lower()
if "de" in pl.split("_") or pl.startswith("01_de") or pl.endswith("_de"):
is_de = True
break
if "_de." in parts_lower or "_de_" in parts_lower:
is_de = True
# Dateien im Root (ohne Sprachordner) sind auch DE
if len(rel.parts) <= 2:
is_de = True
if not is_de:
continue
# Nur Word-erstellte PDFs
if is_word_pdf(pdf):
sources.append(pdf)
return sources
def convert_file(source: Path, input_dir: Path, chapters_dir: Path) -> ConversionResult:
"""Konvertiert eine einzelne Datei und bestimmt Zielkapitel."""
filename = source.name
# Kapitel bestimmen
chapter = detect_chapter(filename)
# Spezialfall: Steuerung -> Unterordner
if chapter == "06_steuerung":
sub = detect_steuerung_subdir(filename)
if sub:
chapter = f"06_steuerung/{sub}"
rst_filename = make_rst_filename(filename)
result = ConversionResult(
source_path=source,
target_chapter=chapter,
rst_filename=rst_filename,
)
# Konvertierung
if source.suffix.lower() == ".docx":
blocks, images, warnings = convert_docx_to_blocks(source)
result.blocks = blocks
result.images = images
result.warnings = warnings
elif source.suffix.lower() == ".pdf":
blocks, warnings = convert_pdf_to_blocks(source)
result.blocks = blocks
result.warnings = warnings
else:
result.warnings.append(f"Unbekanntes Format: {source.suffix}")
return result
def write_result(result: ConversionResult, chapters_dir: Path, dry_run: bool = False) -> None:
"""Schreibt das Konvertierungsergebnis ins Dateisystem."""
target_dir = chapters_dir / result.target_chapter
target_file = target_dir / result.rst_filename
# Titel aus erstem Heading oder Dateiname
title = ""
for block in result.blocks:
if block.kind == "heading" and block.text:
title = block.text
break
if not title:
title = result.source_path.stem
rst_content = render_rst(result.blocks, title="")
if dry_run:
log.info("[DRY-RUN] %s -> %s (%d blocks, %d images)",
result.source_path.name, target_file.relative_to(chapters_dir),
len(result.blocks), len(result.images))
return
# Ordner anlegen
target_dir.mkdir(parents=True, exist_ok=True)
# .rst schreiben
target_file.write_text(rst_content, encoding="utf-8")
log.info("Geschrieben: %s (%d blocks)", target_file.relative_to(chapters_dir), len(result.blocks))
# Bilder speichern
if result.images:
img_dir = target_dir / "images"
img_dir.mkdir(exist_ok=True)
for img_name, img_data in result.images.items():
img_file = img_dir / img_name
img_file.write_bytes(img_data)
log.info(" %d Bilder nach %s", len(result.images), img_dir.relative_to(chapters_dir))
def main() -> None:
parser = argparse.ArgumentParser(
description="Konvertiert Word-Dokumente (.docx) und Word-PDFs nach .rst"
)
parser.add_argument("input_dir", type=Path,
help="Quellverzeichnis mit .docx/.pdf Dateien")
parser.add_argument("output_dir", type=Path,
help="Ziel-chapters/ Verzeichnis")
parser.add_argument("--dry-run", action="store_true",
help="Nur anzeigen, nichts schreiben")
parser.add_argument("--lang", default="de",
help="Zielsprache (default: de)")
parser.add_argument("--verbose", "-v", action="store_true",
help="Ausfuehrliche Ausgabe")
args = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.INFO,
format="%(levelname)-7s %(message)s",
)
input_dir = args.input_dir.resolve()
chapters_dir = args.output_dir.resolve()
if not input_dir.exists():
log.error("Quellverzeichnis nicht gefunden: %s", input_dir)
sys.exit(1)
if not chapters_dir.exists():
log.error("Zielverzeichnis nicht gefunden: %s", chapters_dir)
sys.exit(1)
# Quellen finden
log.info("Suche Quelldateien in %s ...", input_dir)
sources = find_sources(input_dir, args.lang)
log.info("Gefunden: %d Dateien", len(sources))
if not sources:
log.warning("Keine konvertierbaren Dateien gefunden.")
sys.exit(0)
# Anzeige
for src in sources:
rel = src.relative_to(input_dir)
chapter = detect_chapter(src.name)
if chapter == "06_steuerung":
sub = detect_steuerung_subdir(src.name)
if sub:
chapter = f"06_steuerung/{sub}"
log.info(" %s -> %s", rel, chapter)
print()
# Konvertierung
success = 0
errors = 0
for src in sources:
try:
result = convert_file(src, input_dir, chapters_dir)
if result.blocks:
write_result(result, chapters_dir, dry_run=args.dry_run)
success += 1
else:
log.warning("Keine Inhalte extrahiert: %s", src.name)
errors += 1
for w in result.warnings:
log.warning(" %s: %s", src.name, w)
except Exception as e:
log.error("Fehler bei %s: %s", src.name, e)
errors += 1
print()
log.info("Fertig: %d erfolgreich, %d Fehler", success, errors)
if __name__ == "__main__":
main()