Nummernkreise für alle Produkte aus das Produktdatenblatt extrahiert. Fortna als Beispiel .xlsx unter examples dazu. create_examplex.py so impl., dass aus dem Sivas Export eine Liste der wirklich verbauten Produkte extrahiert werden kann, wenn man die cfg Produktnummern richtig pflegt

bin/create_examples.bat

@@ -0,0 +1,20 @@
+@echo off
+REM ================================================================
+REM DOCU_BUILD - Bilder und Text aus Produktprogramm-PDFs extrahieren
+REM ================================================================
+
+call "%~dp0setenv.bat"
+
+if not exist "%PROJECT%\.venv" (
+    echo FEHLER: Virtual environment nicht gefunden.
+    echo Bitte zuerst bin\install_py.bat ausfuehren.
+    exit /b 1
+)
+
+call "%PROJECT%\.venv\Scripts\activate.bat"
+
+echo.
+echo Extrahiere Bilder und Text aus Produktprogramm-PDFs ...
+echo.
+
+python "%PV_LIB%\create_examples.py" --example-dir fortna %*

lib/create_examples.py

@@ -8,8 +8,12 @@ Verwendung:
 from __future__ import annotations
 
 import argparse
+import csv
+import json
 import logging
+import re
 import sys
+from collections import defaultdict
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
@@ -21,6 +25,7 @@ log = logging.getLogger(__name__)
 class StkItem:
     position: str
     teilenummer: str
+    bezeichnung: str
     menge: Any
 
 
@@ -47,7 +52,7 @@ def _extract_stk_liste_ist(xlsx_path: Path) -> list[StkItem]:
 
     ws = wb["Stk.-Liste IST"]
 
-    header_idx: tuple[int | None, int | None, int | None] | None = None
+    header_idx: tuple[int | None, int | None, int | None, int | None] | None = None
     items: list[StkItem] = []
 
     for row in ws.iter_rows(values_only=True):
@@ -57,30 +62,230 @@ def _extract_stk_liste_ist(xlsx_path: Path) -> list[StkItem]:
                 list(row),
                 ["teilenummer", "teilnummer", "article", "artikelnummer"],
             )
+            desc_idx = _find_col_index(
+                list(row),
+                ["bezeichnung", "beschreibung", "description", "benennung"],
+            )
             qty_idx = _find_col_index(list(row), ["menge", "anzahl", "qty", "quantity"])
-            if pos_idx is not None and part_idx is not None and qty_idx is not None:
-                header_idx = (pos_idx, part_idx, qty_idx)
+            if (
+                pos_idx is not None
+                and part_idx is not None
+                and desc_idx is not None
+                and qty_idx is not None
+            ):
+                header_idx = (pos_idx, part_idx, desc_idx, qty_idx)
             continue
 
-        pos_idx, part_idx, qty_idx = header_idx
-        assert pos_idx is not None and part_idx is not None and qty_idx is not None
+        pos_idx, part_idx, desc_idx, qty_idx = header_idx
+        assert (
+            pos_idx is not None
+            and part_idx is not None
+            and desc_idx is not None
+            and qty_idx is not None
+        )
 
         position = _cell_str(row[pos_idx]) if pos_idx < len(row) else ""
         teilenummer = _cell_str(row[part_idx]) if part_idx < len(row) else ""
+        bezeichnung = _cell_str(row[desc_idx]) if desc_idx < len(row) else ""
         menge = row[qty_idx] if qty_idx < len(row) else None
 
         # Leere Zeilen ueberspringen
-        if not position and not teilenummer and menge in ("", None):
+        if not position and not teilenummer and not bezeichnung and menge in ("", None):
             continue
         if not position or not teilenummer:
             continue
 
-        items.append(StkItem(position=position, teilenummer=teilenummer, menge=menge))
+        items.append(
+            StkItem(
+                position=position,
+                teilenummer=teilenummer,
+                bezeichnung=bezeichnung,
+                menge=menge,
+            )
+        )
 
     wb.close()
     return items
 
-def _process_single_folder(folder: Path, example_dir: Path, result: dict, log) -> None:
+
+def _to_number(value: Any) -> float:
+    if value is None:
+        return 0.0
+    if isinstance(value, (int, float)):
+        return float(value)
+    text = str(value).strip().replace(",", ".")
+    if not text:
+        return 0.0
+    try:
+        return float(text)
+    except ValueError:
+        return 0.0
+
+
+def _common_path_prefix(paths: list[str]) -> str:
+    parts_list = [p.split("/") for p in paths if p]
+    if not parts_list:
+        return ""
+    common = parts_list[0]
+    for parts in parts_list[1:]:
+        i = 0
+        max_i = min(len(common), len(parts))
+        while i < max_i and common[i] == parts[i]:
+            i += 1
+        common = common[:i]
+        if not common:
+            break
+    return "/".join(common)
+
+
+def _load_product_numbers(cfg_path: Path) -> dict[str, str]:
+    """
+    Laedt 9-stellige Produktnummern aus cfg/product_numbers.cfg.
+
+    Erwartetes Format:
+      [Sektion]
+      123456789
+      987654321
+    """
+    allowed_sections = {"omniflo", "ils", "cpc", "trolley", "geruest"}
+    products: dict[str, str] = {}
+    current_section = ""
+
+    if not cfg_path.exists():
+        return products
+
+    for raw_line in cfg_path.read_text(encoding="utf-8").splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#"):
+            continue
+
+        if line.startswith("[") and line.endswith("]"):
+            current_section = line[1:-1].strip().lower()
+            continue
+
+        if current_section not in allowed_sections:
+            continue
+
+        if re.match(r"^\d{9}$", line):
+            products[line] = current_section
+
+    return products
+
+
+def write_results(
+    xlsx_path: Path,
+    filtered_items: list[dict[str, Any]],
+    log: logging.Logger,
+    write_json: bool = False,
+    write_csv: bool = False,
+    write_xlsx: bool = False,
+) -> None:
+    if write_json:
+        json_path = xlsx_path.with_suffix(".json")
+        with json_path.open("w", encoding="utf-8") as fh:
+            json.dump(filtered_items, fh, indent=2, ensure_ascii=False, default=str)
+        log.info("JSON geschrieben: %s", json_path)
+
+    if write_csv:
+        csv_path = xlsx_path.with_suffix(".csv")
+        with csv_path.open("w", encoding="utf-8-sig", newline="") as fh:
+            writer = csv.writer(fh, delimiter=";")
+            writer.writerow(["position", "teilenummer", "bezeichnung", "menge"])
+            for row in filtered_items:
+                writer.writerow(
+                    [
+                        row.get("position", ""),
+                        row.get("teilenummer", ""),
+                        row.get("bezeichnung", ""),
+                        row.get("menge", ""),
+                    ]
+                )
+        log.info("CSV geschrieben: %s", csv_path)
+
+    if write_xlsx:
+        import openpyxl
+
+        xlsx_out = xlsx_path.with_name(f"{xlsx_path.stem}_result.xlsx")
+        wb = openpyxl.Workbook()
+        ws_3 = wb.active
+        ws_3.title = "3er"
+        ws_4 = wb.create_sheet("4er")
+        ws_result = wb.create_sheet("Result")
+        ws_product = wb.create_sheet("product")
+
+        headers = ["position", "teilenummer", "bezeichnung", "menge"]
+        ws_3.append(headers)
+        ws_4.append(headers)
+
+        aggregated: dict[str, dict[str, Any]] = defaultdict(
+            lambda: {
+                "teilenummer": "",
+                "bezeichnung": "",
+                "gesamtmenge": 0.0,
+                "positions": [],
+            }
+        )
+
+        for row in filtered_items:
+            slash_count = str(row.get("position", "")).count("/")
+            if slash_count == 3:
+                ws_3.append([row.get(h, "") for h in headers])
+            elif slash_count == 4:
+                ws_4.append([row.get(h, "") for h in headers])
+
+            tn = str(row.get("teilenummer", "")).strip()
+            if not tn:
+                continue
+            if not aggregated[tn]["teilenummer"]:
+                aggregated[tn]["teilenummer"] = tn
+                aggregated[tn]["bezeichnung"] = row.get("bezeichnung", "")
+            aggregated[tn]["gesamtmenge"] += _to_number(row.get("menge"))
+            aggregated[tn]["positions"].append(str(row.get("position", "")).strip())
+
+        ws_result.append(["teilenummer", "bezeichnung", "gesamtmenge", "mainpath", "endpath"])
+
+        project_root = Path(__file__).resolve().parent.parent
+        product_cfg = project_root / "cfg" / "product_numbers.cfg"
+        product_numbers = _load_product_numbers(product_cfg)
+        ws_product.append(["teilenummer", "bezeichnung", "gesamtmenge", "mainpath", "endpath"])
+
+        for tn in sorted(aggregated.keys()):
+            positions = [p for p in aggregated[tn]["positions"] if p]
+            mainpath = _common_path_prefix(positions)
+            variable_parts: list[str] = []
+            for pos in positions:
+                if mainpath:
+                    suffix = pos[len(mainpath):].lstrip("/")
+                else:
+                    suffix = pos
+                if suffix and suffix not in variable_parts:
+                    variable_parts.append(suffix)
+            endpath = ",".join(variable_parts)
+            result_row = [
+                aggregated[tn]["teilenummer"],
+                aggregated[tn]["bezeichnung"],
+                aggregated[tn]["gesamtmenge"],
+                mainpath,
+                endpath,
+            ]
+            ws_result.append(result_row)
+            if tn in product_numbers:
+                ws_product.append(result_row)
+
+        wb.save(str(xlsx_out))
+        wb.close()
+        log.info("XLSX geschrieben: %s", xlsx_out)
+
+
+def _process_single_folder(
+    folder: Path,
+    example_dir: Path,
+    result: dict[str, list[dict[str, Any]]],
+    log: logging.Logger,
+    write_json: bool = False,
+    write_csv: bool = False,
+    write_xlsx: bool = False,
+) -> None:
     xlsx_files = sorted(folder.glob("*.xlsx"))
     if not xlsx_files:
         return
@@ -92,17 +297,34 @@ def _process_single_folder(folder: Path, example_dir: Path, result: dict, log) -
     except ValueError as exc:
         log.warning("%s/%s uebersprungen: %s", rel_folder, xlsx_path.name, exc)
         return
+    size_path = (3, 4)
 
-    result[rel_folder] = [
+    filtered_items = [
         {
             "position": item.position,
             "teilenummer": item.teilenummer,
+            "bezeichnung": item.bezeichnung,
             "menge": item.menge,
         }
         for item in items
+        if item.position.count("/") in size_path
     ]
+    result[rel_folder] = filtered_items
+    write_results(
+        xlsx_path,
+        filtered_items,
+        log,
+        write_json=write_json,
+        write_csv=write_csv,
+        write_xlsx=write_xlsx,
+    )
 
-def process_all(example_dir: Path) -> dict[str, list[dict[str, Any]]]:
+def process_all(
+    example_dir: Path,
+    write_json: bool = False,
+    write_csv: bool = False,
+    write_xlsx: bool = False,
+) -> dict[str, list[dict[str, Any]]]:
     """
     Durchsucht rekursiv alle Unterverzeichnisse in example_dir.
 
@@ -123,7 +345,15 @@ def process_all(example_dir: Path) -> dict[str, list[dict[str, Any]]]:
     result: dict[str, list[dict[str, Any]]] = {}
     for folder in subdirs:
         log.info("Verarbeite %s", folder)
-        _process_single_folder(folder, example_dir, result, log)
+        _process_single_folder(
+            folder,
+            example_dir,
+            result,
+            log,
+            write_json=write_json,
+            write_csv=write_csv,
+            write_xlsx=write_xlsx,
+        )
    
     log.info("Fertig. Gelesene Verzeichnisse mit Daten: %d", len(result))
     return result
@@ -144,6 +374,24 @@ def main() -> None:
     )
     parser.add_argument("--verbose", "-v", action="store_true",
                         help="Ausfuehrliche Ausgabe")
+    parser.add_argument(
+        "--2json",
+        dest="to_json",
+        action="store_true",
+        help="Schreibt Ausgabe als .json neben die jeweilige .xlsx",
+    )
+    parser.add_argument(
+        "--2csv",
+        dest="to_csv",
+        action="store_true",
+        help="Schreibt Ausgabe als .csv neben die jeweilige .xlsx",
+    )
+    parser.add_argument(
+        "--2xlsx",
+        dest="to_xlsx",
+        action="store_true",
+        help="Schreibt Ausgabe als _result.xlsx neben die jeweilige .xlsx",
+    )
     args = parser.parse_args()
 
     logging.basicConfig(
@@ -165,8 +413,14 @@ def main() -> None:
 
     if args.example_dir is None:
         log.info("Beispiel-Verzeichnis (alle Unterordner): %s", examples_root)
-        data = process_all(examples_root)
+        data = process_all(
+            examples_root,
+            write_json=args.to_json,
+            write_csv=args.to_csv,
+            write_xlsx=args.to_xlsx,
+        )
         log.info("Objekt erzeugt mit %d Schluesseln", len(data))
+        print(json.dumps(data, indent=2, ensure_ascii=False, default=str))
         return
 
     # --example-dir: entweder direkter Verzeichnispfad oder Unterordnername in PV_EXAMPLES
@@ -189,8 +443,17 @@ def main() -> None:
 
     log.info("Beispiel-Verzeichnis (einzeln): %s", target_dir)
     result: dict[str, list[dict[str, Any]]] = {}
-    _process_single_folder(target_dir, target_dir.parent, result, log)
+    _process_single_folder(
+        target_dir,
+        target_dir.parent,
+        result,
+        log,
+        write_json=args.to_json,
+        write_csv=args.to_csv,
+        write_xlsx=args.to_xlsx,
+    )
     log.info("Objekt erzeugt mit %d Schluesseln", len(result))
+    #print(json.dumps(result, indent=2, ensure_ascii=False, default=str))
 
 
 if __name__ == "__main__":

lib/extract_products.py

@@ -81,19 +81,17 @@ def _extract_toc_numbers(doc) -> dict[str, str]:
     """Extrahiert Kapitelnummern aus dem Inhaltsverzeichnis des PDFs.
 
     Sucht TOC-Seiten (Seiten mit '....' Punkt-Fuellern) und ordnet
-    Ueberschriftentexte ihren Kapitelnummern zu.
+    Ueberschriftentexte ihren Kapitelnummern zu. Nur Eintraege mit
+    expliziter Nummer werden erfasst (bis Level 2).
 
     Returns:
-        Dict title_text -> kapitelnummer, z.B. {"Typen von Weichen mit Bögen": "2.3.1"}
+        Dict title_text -> kapitelnummer, z.B. {"Weichen Omniflo": "2.3"}
     """
-    import fitz
-
     toc_entries: dict[str, str] = {}
 
     for page_num in range(len(doc)):
         page = doc[page_num]
         text = page.get_text()
-        # Nur TOC-Seiten verarbeiten (enthalten "....")
         if "...." not in text:
             continue
 
@@ -106,8 +104,7 @@ def _extract_toc_numbers(doc) -> dict[str, str]:
             if not line:
                 continue
 
-            # TOC-Zeile: "Text ........... Seitennr"
-            # Kapitelnummer steht oft auf separater Zeile davor
+            # Kapitelnummer auf separater Zeile
             num_match = re.match(r"^(\d+(?:\.\d+)*)\s*$", line)
             if num_match:
                 current_number = num_match.group(1)
@@ -115,32 +112,59 @@ def _extract_toc_numbers(doc) -> dict[str, str]:
 
             # Zeile mit Punkt-Fuellern = TOC-Eintrag
             dot_match = re.search(r"\.{4,}", line)
-            if dot_match:
-                # Text vor den Punkten extrahieren
+            if not dot_match:
+                continue
+
             title = line[:dot_match.start()].strip()
-                # Seitennummer nach den Punkten
-                if title and current_number:
-                    toc_entries[title] = current_number
-                    current_number = ""
-                elif title:
-                    # Nummer koennte am Anfang des Titels stehen
+            if not title:
+                continue
+
+            # Inline-Nummer am Anfang? "1.4 Abkürzungen ....."
             nm = re.match(r"^(\d+(?:\.\d+)*)\s+(.+)", title)
             if nm:
                 toc_entries[nm.group(2).strip()] = nm.group(1)
-
-        # Auch inline Nummern: "1.4 Abkürzungen ..... 8"
-        for line in lines:
-            line = line.strip()
-            dot_match = re.search(r"\.{4,}", line)
-            if dot_match:
-                title_part = line[:dot_match.start()].strip()
-                nm = re.match(r"^(\d+(?:\.\d+)*)\s+(.+)", title_part)
-                if nm:
-                    toc_entries[nm.group(2).strip()] = nm.group(1)
+                current_number = ""
+            elif current_number:
+                toc_entries[title] = current_number
+                current_number = ""
 
     return toc_entries
 
 
+RE_DATE = re.compile(r"^\d{2}\.\d{2}\.\d{4}$")
+RE_COPYRIGHT = re.compile(r"^Copyright\s+\d{4}", re.IGNORECASE)
+RE_VERSION_LINE = re.compile(r"^Version\s*[:.]?\s*\d", re.IGNORECASE)
+
+# Bekannte Rausch-Muster: Autorinnen, Metadaten, Copyright-Zeilen
+_NOISE_KEYWORDS = {
+    "annette schopper", "erstellt:", "freigabe:", "nicht freigegeben",
+    "nur zur information", "schönenberger systeme gmbh",
+    "justus-von-liebig-str", "info@schoenenberger.de",
+    "www.schoenenberger.de",
+}
+
+
+def _is_noise(text: str) -> bool:
+    """Erkennt Rausch-Zeilen die nicht ins RST gehoeren."""
+    lower = text.lower().strip()
+    if RE_COPYRIGHT.match(text):
+        return True
+    if RE_DATE.match(text.strip()):
+        return True
+    if RE_VERSION_LINE.match(text):
+        return True
+    for kw in _NOISE_KEYWORDS:
+        if kw in lower:
+            return True
+    # Seitenzahlen
+    if re.match(r"^Seite\s+\d+$", text, re.IGNORECASE):
+        return True
+    # Einzelne kurze Jahreszahlen mit Autorin: "2020 Annette Schopper"
+    if re.match(r"^\d{4}\s+[A-Z][a-z]+\s+[A-Z][a-z]+$", text):
+        return True
+    return False
+
+
 def extract_text_to_rst(pdf_path: Path, rst_path: Path) -> int:
     """Extrahiert Text aus PDF und schreibt ihn als .rst.
 
@@ -188,6 +212,8 @@ def extract_text_to_rst(pdf_path: Path, rst_path: Path) -> int:
                 text = text.strip()
                 if not text:
                     continue
+                if _is_noise(text):
+                    continue
 
                 raw_lines.append({
                     "text": text,
@@ -311,6 +337,10 @@ def extract_text_to_rst(pdf_path: Path, rst_path: Path) -> int:
     # Mehrfache Leerzeilen reduzieren
     rst = "\n".join(rst_lines)
     rst = re.sub(r"\n{3,}", "\n\n", rst)
+    # 9-stellige Nummern mit Leerzeichen zusammenfuegen: "834 372 007" -> "834372007"
+    rst = re.sub(r"\b(\d{3})\s(\d{3})\s(\d{3})\b", r"\1\2\3", rst)
+    rst = re.sub(r"\b(\d{3})\s(\d{6})\b", r"\1\2", rst)
+    rst = re.sub(r"\b(\d{6})\s(\d{3})\b", r"\1\2", rst)
     rst = rst.strip() + "\n"
 
     rst_path.parent.mkdir(parents=True, exist_ok=True)