m.stangl/doc_build
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

extract_ids.py impl. Bauen der json Files so dass die Bezeichnungen immer mit dabei stehen ausser in den Blättern.

Browse Source
This commit is contained in:
Michael Stangl
2026-05-19 16:19:01 +02:00
parent d10d7c8e01
commit 038b51d35d
4 changed files with 245 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/setenv.bat
+15 -15
View File
@@ -66,21 +66,21 @@ echo ================================================================
echo DOCU_BUILD ENVIRONMENT SETUP COMPLETE
echo ================================================================
echo PROJECT = %PROJECT%
echo PV_BIN = %PV_BIN%
echo PV_CFG = %PV_CFG%
echo PV_LIB = %PV_LIB%
echo PV_DATA = %PV_DATA%
echo PV_RESULTS = %PV_RESULTS%
echo PV_LOG = %PV_LOG%
echo PV_EXAMPLES = %PV_EXAMPLES%
echo PV_EXT = %PV_EXT%
echo PV_TEMPLATES = %PV_TEMPLATES%
echo PV_ASSETS = %PV_ASSETS%
echo PV_MODULES = %PV_MODULES%
echo PV_CHAPTERS = %PV_CHAPTERS%
echo PV_GLOSSARY = %PV_GLOSSARY%
echo PV_STYLES = %PV_STYLES%
echo PV_SCRIPTS = %PV_SCRIPTS%
REM echo PV_BIN = %PV_BIN%
REM echo PV_CFG = %PV_CFG%
REM echo PV_LIB = %PV_LIB%
REM echo PV_DATA = %PV_DATA%
REM echo PV_RESULTS = %PV_RESULTS%
REM echo PV_LOG = %PV_LOG%
REM echo PV_EXAMPLES = %PV_EXAMPLES%
REM echo PV_EXT = %PV_EXT%
REM echo PV_TEMPLATES = %PV_TEMPLATES%
REM echo PV_ASSETS = %PV_ASSETS%
REM echo PV_MODULES = %PV_MODULES%
REM echo PV_CHAPTERS = %PV_CHAPTERS%
REM echo PV_GLOSSARY = %PV_GLOSSARY%
REM echo PV_STYLES = %PV_STYLES%
REM echo PV_SCRIPTS = %PV_SCRIPTS%
echo PYTHONPATH = %PYTHONPATH%
echo ================================================================
echo.
lib/extract_ids.py
+137
View File
@@ -0,0 +1,137 @@
"""Extract 9-digit Sivas IDs from JSON tree files generated by jsonFromXlsx.py.
Collects Sivas numbers that sit one hierarchy level above the leaves
and writes them as 'nummer: bezeichnung' into a text file.
Usage:
python extract_ids.py --filename KA135776
python extract_ids.py --filename KA135776 --tab soll
python extract_ids.py --all
"""
import argparse
import json
import os
import re
import sys
from pathlib import Path
RE_SIVAS = re.compile(r"^(\d{9})")
def is_leaf_value(v) -> bool:
"""True if v is a leaf entry (string, anzahl-dict, or laenge-dict)."""
if isinstance(v, str):
return True
if isinstance(v, dict) and ("anzahl" in v or "laenge" in v):
return True
return False
def extract_ids_from_tree(tree: dict) -> dict[str, str]:
"""Collect 9-digit Sivas numbers one level above leaves.
Returns dict of nummer -> bezeichnung.
"""
ids: dict[str, str] = {}
def walk(node: dict):
for key, value in node.items():
if not isinstance(value, dict) or "anzahl" in value or "laenge" in value:
continue # leaf, skip
# Check if all children are leaves
all_leaves = all(is_leaf_value(v) for v in value.values())
m = RE_SIVAS.match(key)
if all_leaves and value and m:
nummer = m.group(1)
# Bezeichnung from key: part after "nummer-", underscores back to spaces
if len(key) > 9:
bez = key[10:].replace("_", " ")
else:
bez = nummer
ids[nummer] = bez
else:
walk(value)
walk(tree)
return ids
def process_json(json_path: Path, tab: str, results_dir: Path) -> int:
"""Extract IDs from one JSON file. Returns number of IDs found."""
with open(json_path, encoding="utf-8") as f:
tree = json.load(f)
ids = extract_ids_from_tree(tree)
# Derive project name: strip _ist/_soll suffix
project_name = json_path.stem
for suffix in ("_ist", "_soll"):
if project_name.endswith(suffix):
project_name = project_name[: -len(suffix)]
break
ids_path = results_dir / f"{project_name}_ids_{tab}.txt"
with open(ids_path, "w", encoding="utf-8") as f:
for nummer, bez in sorted(ids.items()):
f.write(f"{nummer}: {bez}\n")
print(f" {json_path.name} -> {ids_path.name} ({len(ids)} IDs)")
return len(ids)
def main():
parser = argparse.ArgumentParser(
description="Extrahiert 9-stellige Sivas-Nummern aus JSON-Baeumen im results/-Ordner."
)
parser.add_argument(
"--filename",
default=None,
help="Suchmuster fuer JSON-Dateien im results/-Ordner (z.B. 'KA135776')",
)
parser.add_argument(
"--tab",
choices=["soll", "ist"],
default="ist",
help="Welches Tab gelesen wird: 'ist' oder 'soll' (default: ist)",
)
parser.add_argument(
"--all",
action="store_true",
dest="process_all",
help="Alle *_<tab>.json Dateien im results/-Ordner verarbeiten",
)
args = parser.parse_args()
if not args.process_all and not args.filename:
parser.error("Entweder --filename oder --all muss angegeben werden.")
project_root = Path(os.environ.get("PROJECT", Path(__file__).resolve().parent.parent))
results_dir = project_root / "results"
if not results_dir.exists():
print(f"FEHLER: results/-Ordner nicht gefunden: {results_dir}")
sys.exit(1)
pattern = f"*_{args.tab}.json"
json_files = sorted(results_dir.glob(pattern))
if args.filename:
json_files = [f for f in json_files if args.filename.lower() in f.name.lower()]
if not json_files:
print(f"FEHLER: Keine JSON-Dateien mit Muster '{pattern}' in {results_dir} gefunden.")
if args.filename:
print(f" (Filter: '{args.filename}')")
sys.exit(1)
print(f"Verarbeite {len(json_files)} JSON-Dateien aus {results_dir}\n")
total_ids = 0
for json_path in json_files:
total_ids += process_json(json_path, args.tab, results_dir)
print(f"\nFertig: {total_ids} IDs aus {len(json_files)} Dateien extrahiert.")
if __name__ == "__main__":
main()
lib/extract_products.py
+12 -5
View File
@@ -272,7 +272,12 @@ def extract_text_to_rst(pdf_path: Path, rst_path: Path) -> int:
# --- Phase 3: Ueberschriften erkennen und RST generieren ---
# Heading: Nummerierung wie "1 Text", "2.3 Text", "2.3.1.4 Text"
# Text muss mit einem Buchstaben beginnen (verhindert "610 014 001" etc.)
RE_HEADING = re.compile(r"^(\d+(?:\.\d+)*)\s+([A-Za-zÄÖÜäöüß].+)$")
# Jeder Nummerierungsteil max 3 Stellen (verhindert Datumsangaben wie "10.12.2020")
# Titeltext muss mit einem Wort von mind. 3 Buchstaben beginnen
# (verhindert "2 B. def.", "19.8 mm (LW13)")
RE_HEADING = re.compile(
r"^(\d{1,3}(?:\.\d{1,3})*)\s+([A-Za-zÄÖÜäöüß]{3,}.*)$"
)
rst_lines: list[str] = []
title = pdf_path.stem
rst_lines.append(RST_HEADING_CHARS[1] * len(title))
@@ -310,10 +315,12 @@ def extract_text_to_rst(pdf_path: Path, rst_path: Path) -> int:
toc_num = toc_numbers.get(text)
if toc_num:
# Nummer aus TOC voranstellen
text = f"{toc_num} {text}"
hm = RE_HEADING.match(text)
is_heading = True
log.debug(" TOC-Nummer ergaenzt: %s", text)
candidate = f"{toc_num} {text}"
hm = RE_HEADING.match(candidate)
if hm:
text = candidate
is_heading = True
log.debug(" TOC-Nummer ergaenzt: %s", text)
if is_heading:
num = hm.group(1)
lib/jsonFromXlsx.py
+81 -121
View File
@@ -3,8 +3,7 @@
Usage:
python jsonFromXlsx.py --filename Fortna --tab soll
python jsonFromXlsx.py --filename Fortna --tab ist --output result.json
python jsonFromXlsx.py --filename KA135776 --extract-ids
python jsonFromXlsx.py --from-results --extract-ids
python jsonFromXlsx.py --all
"""
import argparse
@@ -23,11 +22,36 @@ SHEET_NAMES = {
"ist": "Stk.-Liste IST",
}
# Patterns to match sheet names (case-insensitive), ordered by priority
# Matches: Stk.-Liste SOLL, Stck.-Liste Soll, Stückliste SOLL, Stk. Liste IST, etc.
SHEET_PATTERNS = {
"soll": [
re.compile(r"st[ck]+[\.\-]?\s*liste[n]?\s+soll", re.IGNORECASE),
re.compile(r"st[üu]ckliste[n]?\s+soll", re.IGNORECASE),
],
"ist": [
re.compile(r"st[ck]+[\.\-]?\s*liste[n]?\s+ist", re.IGNORECASE),
re.compile(r"st[üu]ckliste[n]?\s+ist", re.IGNORECASE),
],
}
# Column indices (0-based) in the header row
COL_POSITION = 0
COL_TEILENUMMER = 1
COL_BEZEICHNUNG = 2
HEADER_ROW = 3 # 1-based row number of the header
COL_MENGE = 3
HEADER_ROW = 3 # 1-based row number of the header (minimum, actual may be later)
RE_POSITION = re.compile(r"^\d+(/\d+)*$") # valid position pattern: "10" or "10/5/1"
# Sivas numbers where Menge represents length (meters) instead of piece count
LAENGENABHAENGIG = {
"821106001", "821106002",
"825084000", "825084007",
"825104000", "825104007",
"825144000", "825144007",
"821006026", "821006034", "821006035", "821006045",
"825004000",
}
def find_xlsx(examples_dir: str, pattern: str) -> Path:
@@ -78,15 +102,20 @@ def build_tree(ws, sheet_type: str) -> dict:
pos = row[COL_POSITION]
teil = row[COL_TEILENUMMER]
bez = row[COL_BEZEICHNUNG]
menge = row[COL_MENGE] if len(row) > COL_MENGE else None
if pos is None or teil is None:
continue
pos_str = str(pos).strip()
if pos_str.lower() == "position":
continue # skip header row
if not RE_POSITION.match(pos_str):
continue # skip header rows and non-position entries
teil_str = str(teil).strip()
bez_str = str(bez).strip() if bez else ""
try:
menge_val = float(menge) if menge is not None else 0.0
except (ValueError, TypeError):
menge_val = 0.0
teil_counter[teil_str] += 1
rows_data.append((pos_str, teil_str, bez_str))
rows_data.append((pos_str, teil_str, bez_str, menge_val))
unique_teile = {t for t, count in teil_counter.items() if count == 1}
@@ -94,9 +123,9 @@ def build_tree(ws, sheet_type: str) -> dict:
tree: dict = {}
pos_to_node: dict[str, dict] = {} # maps position path -> dict node
# Track metadata per node for leaf post-processing
node_meta: dict[int, tuple[str, str]] = {} # id(child) -> (teil_str, bez_str)
node_meta: dict[int, tuple[str, str, float]] = {} # id(child) -> (teil_str, bez_str, menge)
for pos_str, teil_str, bez_str in rows_data:
for pos_str, teil_str, bez_str, menge_val in rows_data:
key = make_key(teil_str, bez_str, teil_str in unique_teile)
segments = pos_str.split("/")
@@ -115,7 +144,7 @@ def build_tree(ws, sheet_type: str) -> dict:
child: dict = {}
parent[final_key] = child
pos_to_node[pos_str] = child
node_meta[id(child)] = (teil_str, bez_str)
node_meta[id(child)] = (teil_str, bez_str, menge_val)
is_sivas = re.compile(r"^\d{9}$").match
@@ -125,8 +154,17 @@ def build_tree(ws, sheet_type: str) -> dict:
for key, value in node.items():
if isinstance(value, dict) and len(value) == 0:
# Leaf node: use only Teilenummer as key, Bezeichnung as value
teil, bez = node_meta.get(id(value), ("", ""))
if teil in result and is_sivas(teil):
teil, bez, menge = node_meta.get(id(value), ("", "", 0.0))
if teil in LAENGENABHAENGIG:
# Length-dependent part: sum up Menge as laenge
if teil in result and isinstance(result[teil], dict) and "laenge" in result[teil]:
result[teil]["laenge"] += menge
elif teil in result and isinstance(result[teil], str):
result[teil] = {"laenge": menge, "beschreibung": result[teil]}
else:
leaf_key = unique_key(result, teil)
result[leaf_key] = {"laenge": menge, "beschreibung": bez}
elif teil in result and is_sivas(teil):
# Aggregate duplicate 9-digit Sivas numbers
existing = result[teil]
if isinstance(existing, str):
@@ -137,7 +175,16 @@ def build_tree(ws, sheet_type: str) -> dict:
leaf_key = unique_key(result, teil)
result[leaf_key] = bez
elif isinstance(value, dict):
result[key] = simplify_leaves(value)
# Branch node: enrich bare 9-digit keys with Bezeichnung
branch_key = key
if is_sivas(key):
teil, bez, _ = node_meta.get(id(value), ("", "", 0.0))
if bez:
branch_key = f"{teil}-{bez}".replace(" ", "_")
branch_key = unique_key(result, branch_key)
else:
branch_key = unique_key(result, key)
result[branch_key] = simplify_leaves(value)
else:
result[key] = value
return result
@@ -145,49 +192,6 @@ def build_tree(ws, sheet_type: str) -> dict:
return simplify_leaves(tree)
RE_SIVAS = re.compile(r"^(\d{9})")
def is_leaf_value(v) -> bool:
"""True if v is a leaf entry (string or anzahl-dict)."""
if isinstance(v, str):
return True
if isinstance(v, dict) and "anzahl" in v:
return True
return False
def extract_ids_from_tree(tree: dict) -> dict[str, str]:
"""Collect 9-digit Sivas numbers one level above leaves.
Returns dict of nummer -> bezeichnung.
"""
ids: dict[str, str] = {}
def walk(node: dict):
for key, value in node.items():
if not isinstance(value, dict) or "anzahl" in value:
continue # leaf, skip
# Check if all children are leaves
all_leaves = all(is_leaf_value(v) for v in value.values())
m = RE_SIVAS.match(key)
if all_leaves and value and m:
nummer = m.group(1)
# Bezeichnung from key: part after "nummer-", underscores back to spaces
if "-" in key[9:]:
bez = key[10:].replace("_", " ")
elif len(key) > 9:
bez = key[10:].replace("_", " ")
else:
bez = nummer
ids[nummer] = bez
else:
walk(value)
walk(tree)
return ids
def process_file(xlsx_path: Path, sheet_name: str, tab: str, results_dir: Path,
out_path: Path | None = None) -> bool:
"""Process a single xlsx file. Returns True on success."""
@@ -195,11 +199,25 @@ def process_file(xlsx_path: Path, sheet_name: str, tab: str, results_dir: Path,
print(f"Sheet: {sheet_name}")
wb = openpyxl.load_workbook(str(xlsx_path), data_only=True)
if sheet_name not in wb.sheetnames:
print(f" SKIP: Sheet '{sheet_name}' nicht vorhanden.")
# Find matching sheet: try exact (case-insensitive), then pattern matching
actual_name = None
for name in wb.sheetnames:
if name.lower() == sheet_name.lower():
actual_name = name
break
if actual_name is None:
for pattern in SHEET_PATTERNS.get(tab, []):
for name in wb.sheetnames:
if pattern.search(name):
actual_name = name
break
if actual_name:
break
if actual_name is None:
print(f" SKIP: Kein passendes Sheet fuer '{tab}' gefunden. Vorhanden: {wb.sheetnames}")
return False
ws = wb[sheet_name]
ws = wb[actual_name]
tree = build_tree(ws, tab)
if out_path is None:
@@ -239,22 +257,10 @@ def main():
dest="process_all",
help="Alle .xlsx-Dateien im examples/-Ordner in _ist und _soll JSON konvertieren",
)
parser.add_argument(
"--extract-ids",
action="store_true",
dest="extract_ids",
help="9-stellige Sivas-Nummern eine Ebene ueber den Blaettern extrahieren",
)
parser.add_argument(
"--from-results",
action="store_true",
dest="from_results",
help="JSON-Dateien aus results/ lesen statt aus .xlsx zu erzeugen",
)
args = parser.parse_args()
if not args.process_all and not args.filename and not args.from_results:
parser.error("Entweder --filename, --all oder --from-results muss angegeben werden.")
if not args.process_all and not args.filename:
parser.error("Entweder --filename oder --all muss angegeben werden.")
# Resolve paths relative to project root
project_root = Path(os.environ.get("PROJECT", Path(__file__).resolve().parent.parent))
@@ -262,44 +268,16 @@ def main():
results_dir = project_root / "results"
results_dir.mkdir(exist_ok=True)
# Default tab to "ist" when --extract-ids without explicit --tab
tab = args.tab
if args.extract_ids and args.tab == "soll" and "--tab" not in sys.argv:
tab = "ist"
if args.from_results:
# Read existing JSON files from results/
pattern = f"*_{tab}.json"
json_files = sorted(results_dir.glob(pattern))
if args.filename:
json_files = [f for f in json_files if args.filename.lower() in f.name.lower()]
if not json_files:
print(f"FEHLER: Keine JSON-Dateien mit Muster '{pattern}' in {results_dir} gefunden.")
sys.exit(1)
print(f"Lese {len(json_files)} JSON-Dateien aus {results_dir}\n")
for json_path in json_files:
with open(json_path, encoding="utf-8") as f:
tree = json.load(f)
# Derive project name from filename: strip _ist/_soll.json suffix
project_name = json_path.stem
for suffix in ("_ist", "_soll"):
if project_name.endswith(suffix):
project_name = project_name[: -len(suffix)]
break
if args.extract_ids:
ids = extract_ids_from_tree(tree)
ids_path = results_dir / f"{project_name}_ids_{tab}.txt"
with open(ids_path, "w", encoding="utf-8") as f:
for nummer, bez in sorted(ids.items()):
f.write(f"{nummer}: {bez}\n")
print(f" {json_path.name} -> {ids_path.name} ({len(ids)} IDs)")
elif args.process_all:
if args.process_all:
xlsx_files = sorted(f for f in Path(examples_dir).glob("*.xlsx")
if not f.name.startswith("~$"))
print(f"Verarbeite {len(xlsx_files)} Dateien aus {examples_dir}\n")
ok, skip = 0, 0
tabs = [tab] if tab != "soll" or "--tab" in sys.argv else ["soll", "ist"]
for xlsx_path in xlsx_files:
for t in ("soll", "ist"):
for t in tabs:
sheet_name = SHEET_NAMES[t]
if process_file(xlsx_path, sheet_name, t, results_dir):
ok += 1
@@ -317,27 +295,9 @@ def main():
if not out_path.is_absolute():
out_path = results_dir / out_path
json_out = None
if args.extract_ids:
stem = xlsx_path.stem.replace(" ", "_")
json_out = results_dir / f"{stem}_{tab}.json"
if not process_file(xlsx_path, sheet_name, tab, results_dir, out_path):
sys.exit(1)
if args.extract_ids:
# Read back the generated JSON
read_path = out_path or json_out or results_dir / f"{xlsx_path.stem.replace(' ', '_')}_{tab}.json"
with open(read_path, encoding="utf-8") as f:
tree = json.load(f)
ids = extract_ids_from_tree(tree)
project_name = xlsx_path.stem.replace(" ", "_")
ids_path = results_dir / f"{project_name}_ids_{tab}.txt"
with open(ids_path, "w", encoding="utf-8") as f:
for nummer, bez in sorted(ids.items()):
f.write(f"{nummer}: {bez}\n")
print(f" IDs: {ids_path} ({len(ids)} IDs)")
if __name__ == "__main__":
main()