From 038b51d35d4f108d398f8db1b0fb7a6ca6d2df37 Mon Sep 17 00:00:00 2001 From: Michael Stangl Date: Tue, 19 May 2026 16:19:01 +0200 Subject: [PATCH] =?UTF-8?q?extract=5Fids.py=20impl.=20Bauen=20der=20json?= =?UTF-8?q?=20Files=20so=20dass=20die=20Bezeichnungen=20immer=20mit=20dabe?= =?UTF-8?q?i=20stehen=20ausser=20in=20den=20Bl=C3=A4ttern.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/setenv.bat | 30 +++--- lib/extract_ids.py | 137 +++++++++++++++++++++++++++ lib/extract_products.py | 17 +++- lib/jsonFromXlsx.py | 202 ++++++++++++++++------------------------ 4 files changed, 245 insertions(+), 141 deletions(-) create mode 100644 lib/extract_ids.py diff --git a/bin/setenv.bat b/bin/setenv.bat index e29852e..d4a53c8 100644 --- a/bin/setenv.bat +++ b/bin/setenv.bat @@ -66,21 +66,21 @@ echo ================================================================ echo DOCU_BUILD ENVIRONMENT SETUP COMPLETE echo ================================================================ echo PROJECT = %PROJECT% -echo PV_BIN = %PV_BIN% -echo PV_CFG = %PV_CFG% -echo PV_LIB = %PV_LIB% -echo PV_DATA = %PV_DATA% -echo PV_RESULTS = %PV_RESULTS% -echo PV_LOG = %PV_LOG% -echo PV_EXAMPLES = %PV_EXAMPLES% -echo PV_EXT = %PV_EXT% -echo PV_TEMPLATES = %PV_TEMPLATES% -echo PV_ASSETS = %PV_ASSETS% -echo PV_MODULES = %PV_MODULES% -echo PV_CHAPTERS = %PV_CHAPTERS% -echo PV_GLOSSARY = %PV_GLOSSARY% -echo PV_STYLES = %PV_STYLES% -echo PV_SCRIPTS = %PV_SCRIPTS% +REM echo PV_BIN = %PV_BIN% +REM echo PV_CFG = %PV_CFG% +REM echo PV_LIB = %PV_LIB% +REM echo PV_DATA = %PV_DATA% +REM echo PV_RESULTS = %PV_RESULTS% +REM echo PV_LOG = %PV_LOG% +REM echo PV_EXAMPLES = %PV_EXAMPLES% +REM echo PV_EXT = %PV_EXT% +REM echo PV_TEMPLATES = %PV_TEMPLATES% +REM echo PV_ASSETS = %PV_ASSETS% +REM echo PV_MODULES = %PV_MODULES% +REM echo PV_CHAPTERS = %PV_CHAPTERS% +REM echo PV_GLOSSARY = %PV_GLOSSARY% +REM echo PV_STYLES = %PV_STYLES% +REM echo PV_SCRIPTS = %PV_SCRIPTS% echo PYTHONPATH = %PYTHONPATH% echo ================================================================ echo. diff --git a/lib/extract_ids.py b/lib/extract_ids.py new file mode 100644 index 0000000..f08dd2e --- /dev/null +++ b/lib/extract_ids.py @@ -0,0 +1,137 @@ +"""Extract 9-digit Sivas IDs from JSON tree files generated by jsonFromXlsx.py. + +Collects Sivas numbers that sit one hierarchy level above the leaves +and writes them as 'nummer: bezeichnung' into a text file. + +Usage: + python extract_ids.py --filename KA135776 + python extract_ids.py --filename KA135776 --tab soll + python extract_ids.py --all +""" + +import argparse +import json +import os +import re +import sys +from pathlib import Path + + +RE_SIVAS = re.compile(r"^(\d{9})") + + +def is_leaf_value(v) -> bool: + """True if v is a leaf entry (string, anzahl-dict, or laenge-dict).""" + if isinstance(v, str): + return True + if isinstance(v, dict) and ("anzahl" in v or "laenge" in v): + return True + return False + + +def extract_ids_from_tree(tree: dict) -> dict[str, str]: + """Collect 9-digit Sivas numbers one level above leaves. + + Returns dict of nummer -> bezeichnung. + """ + ids: dict[str, str] = {} + + def walk(node: dict): + for key, value in node.items(): + if not isinstance(value, dict) or "anzahl" in value or "laenge" in value: + continue # leaf, skip + # Check if all children are leaves + all_leaves = all(is_leaf_value(v) for v in value.values()) + m = RE_SIVAS.match(key) + if all_leaves and value and m: + nummer = m.group(1) + # Bezeichnung from key: part after "nummer-", underscores back to spaces + if len(key) > 9: + bez = key[10:].replace("_", " ") + else: + bez = nummer + ids[nummer] = bez + else: + walk(value) + + walk(tree) + return ids + + +def process_json(json_path: Path, tab: str, results_dir: Path) -> int: + """Extract IDs from one JSON file. Returns number of IDs found.""" + with open(json_path, encoding="utf-8") as f: + tree = json.load(f) + + ids = extract_ids_from_tree(tree) + + # Derive project name: strip _ist/_soll suffix + project_name = json_path.stem + for suffix in ("_ist", "_soll"): + if project_name.endswith(suffix): + project_name = project_name[: -len(suffix)] + break + + ids_path = results_dir / f"{project_name}_ids_{tab}.txt" + with open(ids_path, "w", encoding="utf-8") as f: + for nummer, bez in sorted(ids.items()): + f.write(f"{nummer}: {bez}\n") + + print(f" {json_path.name} -> {ids_path.name} ({len(ids)} IDs)") + return len(ids) + + +def main(): + parser = argparse.ArgumentParser( + description="Extrahiert 9-stellige Sivas-Nummern aus JSON-Baeumen im results/-Ordner." + ) + parser.add_argument( + "--filename", + default=None, + help="Suchmuster fuer JSON-Dateien im results/-Ordner (z.B. 'KA135776')", + ) + parser.add_argument( + "--tab", + choices=["soll", "ist"], + default="ist", + help="Welches Tab gelesen wird: 'ist' oder 'soll' (default: ist)", + ) + parser.add_argument( + "--all", + action="store_true", + dest="process_all", + help="Alle *_.json Dateien im results/-Ordner verarbeiten", + ) + args = parser.parse_args() + + if not args.process_all and not args.filename: + parser.error("Entweder --filename oder --all muss angegeben werden.") + + project_root = Path(os.environ.get("PROJECT", Path(__file__).resolve().parent.parent)) + results_dir = project_root / "results" + + if not results_dir.exists(): + print(f"FEHLER: results/-Ordner nicht gefunden: {results_dir}") + sys.exit(1) + + pattern = f"*_{args.tab}.json" + json_files = sorted(results_dir.glob(pattern)) + if args.filename: + json_files = [f for f in json_files if args.filename.lower() in f.name.lower()] + + if not json_files: + print(f"FEHLER: Keine JSON-Dateien mit Muster '{pattern}' in {results_dir} gefunden.") + if args.filename: + print(f" (Filter: '{args.filename}')") + sys.exit(1) + + print(f"Verarbeite {len(json_files)} JSON-Dateien aus {results_dir}\n") + total_ids = 0 + for json_path in json_files: + total_ids += process_json(json_path, args.tab, results_dir) + + print(f"\nFertig: {total_ids} IDs aus {len(json_files)} Dateien extrahiert.") + + +if __name__ == "__main__": + main() diff --git a/lib/extract_products.py b/lib/extract_products.py index addc175..dbd719c 100644 --- a/lib/extract_products.py +++ b/lib/extract_products.py @@ -272,7 +272,12 @@ def extract_text_to_rst(pdf_path: Path, rst_path: Path) -> int: # --- Phase 3: Ueberschriften erkennen und RST generieren --- # Heading: Nummerierung wie "1 Text", "2.3 Text", "2.3.1.4 Text" # Text muss mit einem Buchstaben beginnen (verhindert "610 014 001" etc.) - RE_HEADING = re.compile(r"^(\d+(?:\.\d+)*)\s+([A-Za-zÄÖÜäöüß].+)$") + # Jeder Nummerierungsteil max 3 Stellen (verhindert Datumsangaben wie "10.12.2020") + # Titeltext muss mit einem Wort von mind. 3 Buchstaben beginnen + # (verhindert "2 B. def.", "19.8 mm (LW13)") + RE_HEADING = re.compile( + r"^(\d{1,3}(?:\.\d{1,3})*)\s+([A-Za-zÄÖÜäöüß]{3,}.*)$" + ) rst_lines: list[str] = [] title = pdf_path.stem rst_lines.append(RST_HEADING_CHARS[1] * len(title)) @@ -310,10 +315,12 @@ def extract_text_to_rst(pdf_path: Path, rst_path: Path) -> int: toc_num = toc_numbers.get(text) if toc_num: # Nummer aus TOC voranstellen - text = f"{toc_num} {text}" - hm = RE_HEADING.match(text) - is_heading = True - log.debug(" TOC-Nummer ergaenzt: %s", text) + candidate = f"{toc_num} {text}" + hm = RE_HEADING.match(candidate) + if hm: + text = candidate + is_heading = True + log.debug(" TOC-Nummer ergaenzt: %s", text) if is_heading: num = hm.group(1) diff --git a/lib/jsonFromXlsx.py b/lib/jsonFromXlsx.py index 63b8baa..70ff368 100644 --- a/lib/jsonFromXlsx.py +++ b/lib/jsonFromXlsx.py @@ -3,8 +3,7 @@ Usage: python jsonFromXlsx.py --filename Fortna --tab soll python jsonFromXlsx.py --filename Fortna --tab ist --output result.json - python jsonFromXlsx.py --filename KA135776 --extract-ids - python jsonFromXlsx.py --from-results --extract-ids + python jsonFromXlsx.py --all """ import argparse @@ -23,11 +22,36 @@ SHEET_NAMES = { "ist": "Stk.-Liste IST", } +# Patterns to match sheet names (case-insensitive), ordered by priority +# Matches: Stk.-Liste SOLL, Stck.-Liste Soll, Stückliste SOLL, Stk. Liste IST, etc. +SHEET_PATTERNS = { + "soll": [ + re.compile(r"st[ck]+[\.\-]?\s*liste[n]?\s+soll", re.IGNORECASE), + re.compile(r"st[üu]ckliste[n]?\s+soll", re.IGNORECASE), + ], + "ist": [ + re.compile(r"st[ck]+[\.\-]?\s*liste[n]?\s+ist", re.IGNORECASE), + re.compile(r"st[üu]ckliste[n]?\s+ist", re.IGNORECASE), + ], +} + # Column indices (0-based) in the header row COL_POSITION = 0 COL_TEILENUMMER = 1 COL_BEZEICHNUNG = 2 -HEADER_ROW = 3 # 1-based row number of the header +COL_MENGE = 3 +HEADER_ROW = 3 # 1-based row number of the header (minimum, actual may be later) +RE_POSITION = re.compile(r"^\d+(/\d+)*$") # valid position pattern: "10" or "10/5/1" + +# Sivas numbers where Menge represents length (meters) instead of piece count +LAENGENABHAENGIG = { + "821106001", "821106002", + "825084000", "825084007", + "825104000", "825104007", + "825144000", "825144007", + "821006026", "821006034", "821006035", "821006045", + "825004000", +} def find_xlsx(examples_dir: str, pattern: str) -> Path: @@ -78,15 +102,20 @@ def build_tree(ws, sheet_type: str) -> dict: pos = row[COL_POSITION] teil = row[COL_TEILENUMMER] bez = row[COL_BEZEICHNUNG] + menge = row[COL_MENGE] if len(row) > COL_MENGE else None if pos is None or teil is None: continue pos_str = str(pos).strip() - if pos_str.lower() == "position": - continue # skip header row + if not RE_POSITION.match(pos_str): + continue # skip header rows and non-position entries teil_str = str(teil).strip() bez_str = str(bez).strip() if bez else "" + try: + menge_val = float(menge) if menge is not None else 0.0 + except (ValueError, TypeError): + menge_val = 0.0 teil_counter[teil_str] += 1 - rows_data.append((pos_str, teil_str, bez_str)) + rows_data.append((pos_str, teil_str, bez_str, menge_val)) unique_teile = {t for t, count in teil_counter.items() if count == 1} @@ -94,9 +123,9 @@ def build_tree(ws, sheet_type: str) -> dict: tree: dict = {} pos_to_node: dict[str, dict] = {} # maps position path -> dict node # Track metadata per node for leaf post-processing - node_meta: dict[int, tuple[str, str]] = {} # id(child) -> (teil_str, bez_str) + node_meta: dict[int, tuple[str, str, float]] = {} # id(child) -> (teil_str, bez_str, menge) - for pos_str, teil_str, bez_str in rows_data: + for pos_str, teil_str, bez_str, menge_val in rows_data: key = make_key(teil_str, bez_str, teil_str in unique_teile) segments = pos_str.split("/") @@ -115,7 +144,7 @@ def build_tree(ws, sheet_type: str) -> dict: child: dict = {} parent[final_key] = child pos_to_node[pos_str] = child - node_meta[id(child)] = (teil_str, bez_str) + node_meta[id(child)] = (teil_str, bez_str, menge_val) is_sivas = re.compile(r"^\d{9}$").match @@ -125,8 +154,17 @@ def build_tree(ws, sheet_type: str) -> dict: for key, value in node.items(): if isinstance(value, dict) and len(value) == 0: # Leaf node: use only Teilenummer as key, Bezeichnung as value - teil, bez = node_meta.get(id(value), ("", "")) - if teil in result and is_sivas(teil): + teil, bez, menge = node_meta.get(id(value), ("", "", 0.0)) + if teil in LAENGENABHAENGIG: + # Length-dependent part: sum up Menge as laenge + if teil in result and isinstance(result[teil], dict) and "laenge" in result[teil]: + result[teil]["laenge"] += menge + elif teil in result and isinstance(result[teil], str): + result[teil] = {"laenge": menge, "beschreibung": result[teil]} + else: + leaf_key = unique_key(result, teil) + result[leaf_key] = {"laenge": menge, "beschreibung": bez} + elif teil in result and is_sivas(teil): # Aggregate duplicate 9-digit Sivas numbers existing = result[teil] if isinstance(existing, str): @@ -137,7 +175,16 @@ def build_tree(ws, sheet_type: str) -> dict: leaf_key = unique_key(result, teil) result[leaf_key] = bez elif isinstance(value, dict): - result[key] = simplify_leaves(value) + # Branch node: enrich bare 9-digit keys with Bezeichnung + branch_key = key + if is_sivas(key): + teil, bez, _ = node_meta.get(id(value), ("", "", 0.0)) + if bez: + branch_key = f"{teil}-{bez}".replace(" ", "_") + branch_key = unique_key(result, branch_key) + else: + branch_key = unique_key(result, key) + result[branch_key] = simplify_leaves(value) else: result[key] = value return result @@ -145,49 +192,6 @@ def build_tree(ws, sheet_type: str) -> dict: return simplify_leaves(tree) -RE_SIVAS = re.compile(r"^(\d{9})") - - -def is_leaf_value(v) -> bool: - """True if v is a leaf entry (string or anzahl-dict).""" - if isinstance(v, str): - return True - if isinstance(v, dict) and "anzahl" in v: - return True - return False - - -def extract_ids_from_tree(tree: dict) -> dict[str, str]: - """Collect 9-digit Sivas numbers one level above leaves. - - Returns dict of nummer -> bezeichnung. - """ - ids: dict[str, str] = {} - - def walk(node: dict): - for key, value in node.items(): - if not isinstance(value, dict) or "anzahl" in value: - continue # leaf, skip - # Check if all children are leaves - all_leaves = all(is_leaf_value(v) for v in value.values()) - m = RE_SIVAS.match(key) - if all_leaves and value and m: - nummer = m.group(1) - # Bezeichnung from key: part after "nummer-", underscores back to spaces - if "-" in key[9:]: - bez = key[10:].replace("_", " ") - elif len(key) > 9: - bez = key[10:].replace("_", " ") - else: - bez = nummer - ids[nummer] = bez - else: - walk(value) - - walk(tree) - return ids - - def process_file(xlsx_path: Path, sheet_name: str, tab: str, results_dir: Path, out_path: Path | None = None) -> bool: """Process a single xlsx file. Returns True on success.""" @@ -195,11 +199,25 @@ def process_file(xlsx_path: Path, sheet_name: str, tab: str, results_dir: Path, print(f"Sheet: {sheet_name}") wb = openpyxl.load_workbook(str(xlsx_path), data_only=True) - if sheet_name not in wb.sheetnames: - print(f" SKIP: Sheet '{sheet_name}' nicht vorhanden.") + # Find matching sheet: try exact (case-insensitive), then pattern matching + actual_name = None + for name in wb.sheetnames: + if name.lower() == sheet_name.lower(): + actual_name = name + break + if actual_name is None: + for pattern in SHEET_PATTERNS.get(tab, []): + for name in wb.sheetnames: + if pattern.search(name): + actual_name = name + break + if actual_name: + break + if actual_name is None: + print(f" SKIP: Kein passendes Sheet fuer '{tab}' gefunden. Vorhanden: {wb.sheetnames}") return False - ws = wb[sheet_name] + ws = wb[actual_name] tree = build_tree(ws, tab) if out_path is None: @@ -239,22 +257,10 @@ def main(): dest="process_all", help="Alle .xlsx-Dateien im examples/-Ordner in _ist und _soll JSON konvertieren", ) - parser.add_argument( - "--extract-ids", - action="store_true", - dest="extract_ids", - help="9-stellige Sivas-Nummern eine Ebene ueber den Blaettern extrahieren", - ) - parser.add_argument( - "--from-results", - action="store_true", - dest="from_results", - help="JSON-Dateien aus results/ lesen statt aus .xlsx zu erzeugen", - ) args = parser.parse_args() - if not args.process_all and not args.filename and not args.from_results: - parser.error("Entweder --filename, --all oder --from-results muss angegeben werden.") + if not args.process_all and not args.filename: + parser.error("Entweder --filename oder --all muss angegeben werden.") # Resolve paths relative to project root project_root = Path(os.environ.get("PROJECT", Path(__file__).resolve().parent.parent)) @@ -262,44 +268,16 @@ def main(): results_dir = project_root / "results" results_dir.mkdir(exist_ok=True) - # Default tab to "ist" when --extract-ids without explicit --tab tab = args.tab - if args.extract_ids and args.tab == "soll" and "--tab" not in sys.argv: - tab = "ist" - if args.from_results: - # Read existing JSON files from results/ - pattern = f"*_{tab}.json" - json_files = sorted(results_dir.glob(pattern)) - if args.filename: - json_files = [f for f in json_files if args.filename.lower() in f.name.lower()] - if not json_files: - print(f"FEHLER: Keine JSON-Dateien mit Muster '{pattern}' in {results_dir} gefunden.") - sys.exit(1) - print(f"Lese {len(json_files)} JSON-Dateien aus {results_dir}\n") - for json_path in json_files: - with open(json_path, encoding="utf-8") as f: - tree = json.load(f) - # Derive project name from filename: strip _ist/_soll.json suffix - project_name = json_path.stem - for suffix in ("_ist", "_soll"): - if project_name.endswith(suffix): - project_name = project_name[: -len(suffix)] - break - if args.extract_ids: - ids = extract_ids_from_tree(tree) - ids_path = results_dir / f"{project_name}_ids_{tab}.txt" - with open(ids_path, "w", encoding="utf-8") as f: - for nummer, bez in sorted(ids.items()): - f.write(f"{nummer}: {bez}\n") - print(f" {json_path.name} -> {ids_path.name} ({len(ids)} IDs)") - elif args.process_all: + if args.process_all: xlsx_files = sorted(f for f in Path(examples_dir).glob("*.xlsx") if not f.name.startswith("~$")) print(f"Verarbeite {len(xlsx_files)} Dateien aus {examples_dir}\n") ok, skip = 0, 0 + tabs = [tab] if tab != "soll" or "--tab" in sys.argv else ["soll", "ist"] for xlsx_path in xlsx_files: - for t in ("soll", "ist"): + for t in tabs: sheet_name = SHEET_NAMES[t] if process_file(xlsx_path, sheet_name, t, results_dir): ok += 1 @@ -317,27 +295,9 @@ def main(): if not out_path.is_absolute(): out_path = results_dir / out_path - json_out = None - if args.extract_ids: - stem = xlsx_path.stem.replace(" ", "_") - json_out = results_dir / f"{stem}_{tab}.json" - if not process_file(xlsx_path, sheet_name, tab, results_dir, out_path): sys.exit(1) - if args.extract_ids: - # Read back the generated JSON - read_path = out_path or json_out or results_dir / f"{xlsx_path.stem.replace(' ', '_')}_{tab}.json" - with open(read_path, encoding="utf-8") as f: - tree = json.load(f) - ids = extract_ids_from_tree(tree) - project_name = xlsx_path.stem.replace(" ", "_") - ids_path = results_dir / f"{project_name}_ids_{tab}.txt" - with open(ids_path, "w", encoding="utf-8") as f: - for nummer, bez in sorted(ids.items()): - f.write(f"{nummer}: {bez}\n") - print(f" IDs: {ids_path} ({len(ids)} IDs)") - if __name__ == "__main__": main()