Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6e64a82c70 | |||
| a7bc331a7f |
@@ -0,0 +1,20 @@
|
|||||||
|
@echo off
|
||||||
|
REM ================================================================
|
||||||
|
REM DOCU_BUILD - Bilder und Text aus Produktprogramm-PDFs extrahieren
|
||||||
|
REM ================================================================
|
||||||
|
|
||||||
|
call "%~dp0setenv.bat"
|
||||||
|
|
||||||
|
if not exist "%PROJECT%\.venv" (
|
||||||
|
echo FEHLER: Virtual environment nicht gefunden.
|
||||||
|
echo Bitte zuerst bin\install_py.bat ausfuehren.
|
||||||
|
exit /b 1
|
||||||
|
)
|
||||||
|
|
||||||
|
call "%PROJECT%\.venv\Scripts\activate.bat"
|
||||||
|
|
||||||
|
echo.
|
||||||
|
echo Extrahiere Bilder und Text aus Produktprogramm-PDFs ...
|
||||||
|
echo.
|
||||||
|
|
||||||
|
python "%PV_LIB%\create_examples.py" --example-dir fortna %*
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,162 @@
|
|||||||
# Kernkonzepte
|
# Kernkonzepte
|
||||||
|
|
||||||
|
## Projektübersicht
|
||||||
|
|
||||||
|
Sphinx-basiertes Dokumentationssystem für Transport- und Hängefördersysteme.
|
||||||
|
JSON-gesteuerter Build erzeugt aus modularen Textbausteinen mehrsprachige PDF-Dokumentationen (250+ Seiten).
|
||||||
|
Zwei-Repo-Architektur: allgemeine Ressourcen (Base-Repo) + projektspezifische Anpassungen (Projekt-Repo).
|
||||||
|
|
||||||
|
Auftraggeber: Firma Schönenberger
|
||||||
|
Umsetzung: Mista GmbH
|
||||||
|
|
||||||
|
## Architektur
|
||||||
|
|
||||||
|
### Zwei-Repo-Trennung
|
||||||
|
|
||||||
|
- **transport-docs-base** (Gitea): Alle wiederverwendbaren Inhalte – Module, Kapitelgerüste, Assets, Glossar, Sphinx-Extensions, LaTeX-Templates, Hilfsskripte. Semver-versioniert (VERSION-Datei).
|
||||||
|
- **projekt-kunde-xyz** (pro Anlage): anlage.json (ERP-Export), projektspezifische Bilder, Overrides, Zusatzkapitel, conf.py, build.py. Das Base-Repo wird beim Build als Mount/Share unter `transport-docs-base/` bereitgestellt.
|
||||||
|
|
||||||
|
### Technologie-Stack
|
||||||
|
|
||||||
|
| Komponente | Technologie |
|
||||||
|
|------------------|--------------------------------------------------|
|
||||||
|
| Build-System | Sphinx (Python) |
|
||||||
|
| Textformat | reStructuredText (rST) |
|
||||||
|
| Templating | Jinja2 (Rahmenkapitel, Titelseite) |
|
||||||
|
| PDF-Erzeugung | LaTeX (pdflatex/lualatex) |
|
||||||
|
| Versionierung | Git (Gitea auf Hetzner) |
|
||||||
|
| Konfiguration | JSON (ERP-Export) + YAML (Modul-Metadaten) |
|
||||||
|
| CI/Automatisierung | Gitea Actions oder n8n |
|
||||||
|
| Bildkonvertierung | ezdxf → SVG, Pillow (Optimierung) |
|
||||||
|
|
||||||
|
## Repo-Struktur
|
||||||
|
|
||||||
|
### Base-Repo: transport-docs-base
|
||||||
|
|
||||||
|
```
|
||||||
|
transport-docs-base/
|
||||||
|
├── conf_base.py # Sphinx-Basiskonfiguration (wird importiert)
|
||||||
|
├── VERSION # Semver der Dokumentationsbasis
|
||||||
|
├── _ext/
|
||||||
|
│ ├── anlagen_builder.py # Hauptextension: JSON → Buchstruktur
|
||||||
|
│ ├── localized_figure.py # Sprachauflösung für Bilder
|
||||||
|
│ └── glossary_loader.py # CSV → Glossar-Direktive
|
||||||
|
├── _templates/
|
||||||
|
│ ├── latex/
|
||||||
|
│ │ ├── preamble.tex # Firmen-CI: Fonts, Farben, Kopfzeilen
|
||||||
|
│ │ ├── titlepage.tex # Titelseite-Template (Jinja2)
|
||||||
|
│ │ └── maketitle.tex
|
||||||
|
│ └── common/
|
||||||
|
│ └── header_footer.rst
|
||||||
|
├── assets/
|
||||||
|
│ ├── shared/ # Sprachneutrale Bilder (~90%)
|
||||||
|
│ │ ├── photos/
|
||||||
|
│ │ │ ├── weiche_typ_a_foto_01.jpg
|
||||||
|
│ │ │ ├── hubstation_foto_01.jpg
|
||||||
|
│ │ │ └── antrieb_sew_foto_01.jpg
|
||||||
|
│ │ ├── drawings/ # Technische Zeichnungen (DXF → SVG/PDF)
|
||||||
|
│ │ │ ├── weiche_typ_a_layout.svg
|
||||||
|
│ │ │ └── hubstation_schnitt.pdf
|
||||||
|
│ │ ├── symbols/ # ISO 7010 Sicherheitssymbole
|
||||||
|
│ │ │ ├── W001_warnung.svg
|
||||||
|
│ │ │ └── M004_augenschutz.svg
|
||||||
|
│ │ └── diagrams/ # Schaltpläne, Pneumatik
|
||||||
|
│ │ └── stromlaufplan_weiche_a.pdf
|
||||||
|
│ └── localized/ # Sprachspezifische Bilder
|
||||||
|
│ ├── de/
|
||||||
|
│ │ ├── screenshots/
|
||||||
|
│ │ │ └── hmi_hauptmenu.png
|
||||||
|
│ │ └── diagramme/
|
||||||
|
│ │ └── risikograph_weiche_a.svg
|
||||||
|
│ ├── en/
|
||||||
|
│ │ ├── screenshots/
|
||||||
|
│ │ │ └── hmi_hauptmenu.png
|
||||||
|
│ │ └── diagramme/
|
||||||
|
│ │ └── risikograph_weiche_a.svg
|
||||||
|
│ └── fr/
|
||||||
|
│ └── ...
|
||||||
|
├── modules/ # Textbausteine pro Maschinentyp
|
||||||
|
│ ├── weiche_typ_a/
|
||||||
|
│ │ ├── meta.yaml # Abhängigkeiten + Kapitelzuordnung
|
||||||
|
│ │ ├── beschreibung.de.rst
|
||||||
|
│ │ ├── beschreibung.en.rst
|
||||||
|
│ │ ├── risikomatrix.de.rst
|
||||||
|
│ │ ├── risikomatrix.en.rst
|
||||||
|
│ │ ├── wartung.de.rst
|
||||||
|
│ │ └── images/
|
||||||
|
│ │ └── montage_detail_01.jpg
|
||||||
|
│ ├── hubstation/
|
||||||
|
│ │ ├── meta.yaml
|
||||||
|
│ │ ├── beschreibung.de.rst
|
||||||
|
│ │ ├── technische_daten.de.rst
|
||||||
|
│ │ ├── wartung.de.rst
|
||||||
|
│ │ └── images/
|
||||||
|
│ │ └── hubmechanik_detail.jpg
|
||||||
|
│ ├── antrieb_sew/
|
||||||
|
│ │ ├── meta.yaml
|
||||||
|
│ │ └── ...
|
||||||
|
│ └── steuerung_siemens_s7/
|
||||||
|
│ ├── meta.yaml
|
||||||
|
│ └── ...
|
||||||
|
├── chapters/ # Rahmenkapitel (Gerüst)
|
||||||
|
│ ├── 01_einleitung/
|
||||||
|
│ │ ├── template.de.rst # Jinja2-Template mit Platzhaltern
|
||||||
|
│ │ └── template.en.rst
|
||||||
|
│ ├── 02_sicherheit/
|
||||||
|
│ │ ├── allgemein.de.rst # Immer dabei
|
||||||
|
│ │ ├── allgemein.en.rst
|
||||||
|
│ │ ├── normen_ce.de.rst # EU-Konformität
|
||||||
|
│ │ └── normen_ukca.en.rst # UK-spezifisch
|
||||||
|
│ ├── 03_anlagenbeschreibung/
|
||||||
|
│ │ └── template.de.rst # Hier werden Module eingefügt
|
||||||
|
│ ├── 04_betrieb/
|
||||||
|
│ │ └── template.de.rst
|
||||||
|
│ ├── 05_wartung/
|
||||||
|
│ │ ├── allgemein.de.rst
|
||||||
|
│ │ └── schmierplan_template.de.rst
|
||||||
|
│ ├── 06_ersatzteile/
|
||||||
|
│ │ └── template.de.rst
|
||||||
|
│ └── 07_anhang/
|
||||||
|
│ └── template.de.rst
|
||||||
|
├── glossary/
|
||||||
|
│ ├── terms.de.csv # "Begriff;Definition;Kategorie"
|
||||||
|
│ ├── terms.en.csv
|
||||||
|
│ └── terms.fr.csv
|
||||||
|
├── styles/
|
||||||
|
│ ├── firma_logo.pdf
|
||||||
|
│ ├── firma_logo_sw.pdf
|
||||||
|
│ └── color_scheme.yaml # CI-Farben für LaTeX und SVG
|
||||||
|
├── scripts/
|
||||||
|
│ ├── dxf2svg.py # DXF → SVG Konvertierung
|
||||||
|
│ ├── optimize_images.py # Bildoptimierung vor Commit
|
||||||
|
│ └── validate_modules.py # Prüft meta.yaml Konsistenz
|
||||||
|
```
|
||||||
|
|
||||||
|
### Projekt-Repo: projekt-kunde-xyz
|
||||||
|
|
||||||
|
```
|
||||||
|
projekt-kunde-xyz-2026/
|
||||||
|
├── conf.py # Importiert conf_base.py vom Base-Repo
|
||||||
|
├── anlage.json # ERP-Export: Maschinenauswahl + Metadaten
|
||||||
|
├── build.py # Hauptbuild-Skript
|
||||||
|
├── Makefile
|
||||||
|
├── README.md
|
||||||
|
├── images/ # Projektspezifische Bilder
|
||||||
|
│ ├── aufstellplan_halle3.pdf
|
||||||
|
│ ├── foto_abnahme_01.jpg
|
||||||
|
│ └── kundenlogo.pdf
|
||||||
|
├── overrides/ # Projektspezifische Überschreibungen
|
||||||
|
│ ├── modules/
|
||||||
|
│ │ └── weiche_typ_a/
|
||||||
|
│ │ └── beschreibung.de.rst # Überschreibt Basis wenn vorhanden
|
||||||
|
│ └── chapters/
|
||||||
|
│ └── 01_einleitung/
|
||||||
|
│ └── template.de.rst # Kundenspezifische Einleitung
|
||||||
|
├── additional/ # Kapitel die NUR dieses Projekt hat
|
||||||
|
│ ├── sonderausfuehrung_xyz.de.rst
|
||||||
|
│ └── sonderausfuehrung_xyz.en.rst
|
||||||
|
└── transport-docs-base/ ──────────── # Symlink/Mount zum Base-Repo (Share)
|
||||||
|
```
|
||||||
### anlage.json (ERP-Export)
|
### anlage.json (ERP-Export)
|
||||||
|
|
||||||
Steuert die Zusammenstellung. Enthält Projektmetadaten, Zielsprachen, Normenregion, Liste verbauter Maschinen und Referenzen auf Zusatzkapitel.
|
Steuert die Zusammenstellung. Enthält Projektmetadaten, Zielsprachen, Normenregion, Liste verbauter Maschinen und Referenzen auf Zusatzkapitel.
|
||||||
@@ -220,3 +377,28 @@ Ausgabe:
|
|||||||
- `scripts/dxf2svg.py` – DXF → SVG Konvertierung (ezdxf)
|
- `scripts/dxf2svg.py` – DXF → SVG Konvertierung (ezdxf)
|
||||||
- `scripts/optimize_images.py` – Bildoptimierung vor Commit (Pillow, max 2000px)
|
- `scripts/optimize_images.py` – Bildoptimierung vor Commit (Pillow, max 2000px)
|
||||||
- `scripts/validate_modules.py` – Prüft meta.yaml-Konsistenz, fehlende Dateien/Übersetzungen
|
- `scripts/validate_modules.py` – Prüft meta.yaml-Konsistenz, fehlende Dateien/Übersetzungen
|
||||||
|
|
||||||
|
## Infrastruktur
|
||||||
|
|
||||||
|
- Hetzner-Server: Gitea (Repos), n8n (CI-Automatisierung), Nextcloud (Dateiablage)
|
||||||
|
- Build-Umgebung: Python 3.11+, Sphinx, LaTeX-Distribution (texlive)
|
||||||
|
- CI: Gitea Actions oder n8n-Webhook → Build → PDF nach Nextcloud
|
||||||
|
|
||||||
|
## Umsetzungsphasen
|
||||||
|
|
||||||
|
| Phase | Beschreibung | Aufwand |
|
||||||
|
|-------|---------------------------------------------------------------|-----------|
|
||||||
|
| 1 | PoC: Ein Modul, ein Kapitel, zwei Sprachen, PDF | 2–3 Tage |
|
||||||
|
| 2 | Base-Repo aufsetzen: Sphinx, Extensions, LaTeX-CI | 3–5 Tage |
|
||||||
|
| 3 | Erste 5–10 Module migrieren, Glossar aufbauen | 1–2 Wochen|
|
||||||
|
| 4 | JSON-Export aus ERP definieren, build.py fertigstellen | 2–3 Tage |
|
||||||
|
| 5 | Pilotprojekt: Erste vollständige Anlagendokumentation | 1 Woche |
|
||||||
|
| 6 | CI/CD: Automatischer Build via Gitea Actions oder n8n | 1–2 Tage |
|
||||||
|
|
||||||
|
## Coding-Konventionen
|
||||||
|
|
||||||
|
- Python: Type Hints, Docstrings, pyright-kompatibel
|
||||||
|
- rST: Eine Datei pro Section, Dateiname = Section-Slug
|
||||||
|
- YAML: 2 Spaces Einrückung, keine Tabs
|
||||||
|
- Commits: Conventional Commits (`feat:`, `fix:`, `docs:`)
|
||||||
|
- Bilder: Dateinamen lowercase, Underscores, beschreibend (`weiche_typ_a_foto_01.jpg`)
|
||||||
|
|||||||
Binary file not shown.
+276
-13
@@ -8,8 +8,12 @@ Verwendung:
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import csv
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import re
|
||||||
import sys
|
import sys
|
||||||
|
from collections import defaultdict
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
@@ -21,6 +25,7 @@ log = logging.getLogger(__name__)
|
|||||||
class StkItem:
|
class StkItem:
|
||||||
position: str
|
position: str
|
||||||
teilenummer: str
|
teilenummer: str
|
||||||
|
bezeichnung: str
|
||||||
menge: Any
|
menge: Any
|
||||||
|
|
||||||
|
|
||||||
@@ -47,7 +52,7 @@ def _extract_stk_liste_ist(xlsx_path: Path) -> list[StkItem]:
|
|||||||
|
|
||||||
ws = wb["Stk.-Liste IST"]
|
ws = wb["Stk.-Liste IST"]
|
||||||
|
|
||||||
header_idx: tuple[int | None, int | None, int | None] | None = None
|
header_idx: tuple[int | None, int | None, int | None, int | None] | None = None
|
||||||
items: list[StkItem] = []
|
items: list[StkItem] = []
|
||||||
|
|
||||||
for row in ws.iter_rows(values_only=True):
|
for row in ws.iter_rows(values_only=True):
|
||||||
@@ -57,30 +62,230 @@ def _extract_stk_liste_ist(xlsx_path: Path) -> list[StkItem]:
|
|||||||
list(row),
|
list(row),
|
||||||
["teilenummer", "teilnummer", "article", "artikelnummer"],
|
["teilenummer", "teilnummer", "article", "artikelnummer"],
|
||||||
)
|
)
|
||||||
|
desc_idx = _find_col_index(
|
||||||
|
list(row),
|
||||||
|
["bezeichnung", "beschreibung", "description", "benennung"],
|
||||||
|
)
|
||||||
qty_idx = _find_col_index(list(row), ["menge", "anzahl", "qty", "quantity"])
|
qty_idx = _find_col_index(list(row), ["menge", "anzahl", "qty", "quantity"])
|
||||||
if pos_idx is not None and part_idx is not None and qty_idx is not None:
|
if (
|
||||||
header_idx = (pos_idx, part_idx, qty_idx)
|
pos_idx is not None
|
||||||
|
and part_idx is not None
|
||||||
|
and desc_idx is not None
|
||||||
|
and qty_idx is not None
|
||||||
|
):
|
||||||
|
header_idx = (pos_idx, part_idx, desc_idx, qty_idx)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
pos_idx, part_idx, qty_idx = header_idx
|
pos_idx, part_idx, desc_idx, qty_idx = header_idx
|
||||||
assert pos_idx is not None and part_idx is not None and qty_idx is not None
|
assert (
|
||||||
|
pos_idx is not None
|
||||||
|
and part_idx is not None
|
||||||
|
and desc_idx is not None
|
||||||
|
and qty_idx is not None
|
||||||
|
)
|
||||||
|
|
||||||
position = _cell_str(row[pos_idx]) if pos_idx < len(row) else ""
|
position = _cell_str(row[pos_idx]) if pos_idx < len(row) else ""
|
||||||
teilenummer = _cell_str(row[part_idx]) if part_idx < len(row) else ""
|
teilenummer = _cell_str(row[part_idx]) if part_idx < len(row) else ""
|
||||||
|
bezeichnung = _cell_str(row[desc_idx]) if desc_idx < len(row) else ""
|
||||||
menge = row[qty_idx] if qty_idx < len(row) else None
|
menge = row[qty_idx] if qty_idx < len(row) else None
|
||||||
|
|
||||||
# Leere Zeilen ueberspringen
|
# Leere Zeilen ueberspringen
|
||||||
if not position and not teilenummer and menge in ("", None):
|
if not position and not teilenummer and not bezeichnung and menge in ("", None):
|
||||||
continue
|
continue
|
||||||
if not position or not teilenummer:
|
if not position or not teilenummer:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
items.append(StkItem(position=position, teilenummer=teilenummer, menge=menge))
|
items.append(
|
||||||
|
StkItem(
|
||||||
|
position=position,
|
||||||
|
teilenummer=teilenummer,
|
||||||
|
bezeichnung=bezeichnung,
|
||||||
|
menge=menge,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
wb.close()
|
wb.close()
|
||||||
return items
|
return items
|
||||||
|
|
||||||
def _process_single_folder(folder: Path, example_dir: Path, result: dict, log) -> None:
|
|
||||||
|
def _to_number(value: Any) -> float:
|
||||||
|
if value is None:
|
||||||
|
return 0.0
|
||||||
|
if isinstance(value, (int, float)):
|
||||||
|
return float(value)
|
||||||
|
text = str(value).strip().replace(",", ".")
|
||||||
|
if not text:
|
||||||
|
return 0.0
|
||||||
|
try:
|
||||||
|
return float(text)
|
||||||
|
except ValueError:
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def _common_path_prefix(paths: list[str]) -> str:
|
||||||
|
parts_list = [p.split("/") for p in paths if p]
|
||||||
|
if not parts_list:
|
||||||
|
return ""
|
||||||
|
common = parts_list[0]
|
||||||
|
for parts in parts_list[1:]:
|
||||||
|
i = 0
|
||||||
|
max_i = min(len(common), len(parts))
|
||||||
|
while i < max_i and common[i] == parts[i]:
|
||||||
|
i += 1
|
||||||
|
common = common[:i]
|
||||||
|
if not common:
|
||||||
|
break
|
||||||
|
return "/".join(common)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_product_numbers(cfg_path: Path) -> dict[str, str]:
|
||||||
|
"""
|
||||||
|
Laedt 9-stellige Produktnummern aus cfg/product_numbers.cfg.
|
||||||
|
|
||||||
|
Erwartetes Format:
|
||||||
|
[Sektion]
|
||||||
|
123456789
|
||||||
|
987654321
|
||||||
|
"""
|
||||||
|
allowed_sections = {"omniflo", "ils", "cpc", "trolley", "geruest"}
|
||||||
|
products: dict[str, str] = {}
|
||||||
|
current_section = ""
|
||||||
|
|
||||||
|
if not cfg_path.exists():
|
||||||
|
return products
|
||||||
|
|
||||||
|
for raw_line in cfg_path.read_text(encoding="utf-8").splitlines():
|
||||||
|
line = raw_line.strip()
|
||||||
|
if not line or line.startswith("#"):
|
||||||
|
continue
|
||||||
|
|
||||||
|
if line.startswith("[") and line.endswith("]"):
|
||||||
|
current_section = line[1:-1].strip().lower()
|
||||||
|
continue
|
||||||
|
|
||||||
|
if current_section not in allowed_sections:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if re.match(r"^\d{9}$", line):
|
||||||
|
products[line] = current_section
|
||||||
|
|
||||||
|
return products
|
||||||
|
|
||||||
|
|
||||||
|
def write_results(
|
||||||
|
xlsx_path: Path,
|
||||||
|
filtered_items: list[dict[str, Any]],
|
||||||
|
log: logging.Logger,
|
||||||
|
write_json: bool = False,
|
||||||
|
write_csv: bool = False,
|
||||||
|
write_xlsx: bool = False,
|
||||||
|
) -> None:
|
||||||
|
if write_json:
|
||||||
|
json_path = xlsx_path.with_suffix(".json")
|
||||||
|
with json_path.open("w", encoding="utf-8") as fh:
|
||||||
|
json.dump(filtered_items, fh, indent=2, ensure_ascii=False, default=str)
|
||||||
|
log.info("JSON geschrieben: %s", json_path)
|
||||||
|
|
||||||
|
if write_csv:
|
||||||
|
csv_path = xlsx_path.with_suffix(".csv")
|
||||||
|
with csv_path.open("w", encoding="utf-8-sig", newline="") as fh:
|
||||||
|
writer = csv.writer(fh, delimiter=";")
|
||||||
|
writer.writerow(["position", "teilenummer", "bezeichnung", "menge"])
|
||||||
|
for row in filtered_items:
|
||||||
|
writer.writerow(
|
||||||
|
[
|
||||||
|
row.get("position", ""),
|
||||||
|
row.get("teilenummer", ""),
|
||||||
|
row.get("bezeichnung", ""),
|
||||||
|
row.get("menge", ""),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
log.info("CSV geschrieben: %s", csv_path)
|
||||||
|
|
||||||
|
if write_xlsx:
|
||||||
|
import openpyxl
|
||||||
|
|
||||||
|
xlsx_out = xlsx_path.with_name(f"{xlsx_path.stem}_result.xlsx")
|
||||||
|
wb = openpyxl.Workbook()
|
||||||
|
ws_3 = wb.active
|
||||||
|
ws_3.title = "3er"
|
||||||
|
ws_4 = wb.create_sheet("4er")
|
||||||
|
ws_result = wb.create_sheet("Result")
|
||||||
|
ws_product = wb.create_sheet("product")
|
||||||
|
|
||||||
|
headers = ["position", "teilenummer", "bezeichnung", "menge"]
|
||||||
|
ws_3.append(headers)
|
||||||
|
ws_4.append(headers)
|
||||||
|
|
||||||
|
aggregated: dict[str, dict[str, Any]] = defaultdict(
|
||||||
|
lambda: {
|
||||||
|
"teilenummer": "",
|
||||||
|
"bezeichnung": "",
|
||||||
|
"gesamtmenge": 0.0,
|
||||||
|
"positions": [],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
for row in filtered_items:
|
||||||
|
slash_count = str(row.get("position", "")).count("/")
|
||||||
|
if slash_count == 3:
|
||||||
|
ws_3.append([row.get(h, "") for h in headers])
|
||||||
|
elif slash_count == 4:
|
||||||
|
ws_4.append([row.get(h, "") for h in headers])
|
||||||
|
|
||||||
|
tn = str(row.get("teilenummer", "")).strip()
|
||||||
|
if not tn:
|
||||||
|
continue
|
||||||
|
if not aggregated[tn]["teilenummer"]:
|
||||||
|
aggregated[tn]["teilenummer"] = tn
|
||||||
|
aggregated[tn]["bezeichnung"] = row.get("bezeichnung", "")
|
||||||
|
aggregated[tn]["gesamtmenge"] += _to_number(row.get("menge"))
|
||||||
|
aggregated[tn]["positions"].append(str(row.get("position", "")).strip())
|
||||||
|
|
||||||
|
ws_result.append(["teilenummer", "bezeichnung", "gesamtmenge", "mainpath", "endpath"])
|
||||||
|
|
||||||
|
project_root = Path(__file__).resolve().parent.parent
|
||||||
|
product_cfg = project_root / "cfg" / "product_numbers.cfg"
|
||||||
|
product_numbers = _load_product_numbers(product_cfg)
|
||||||
|
ws_product.append(["teilenummer", "bezeichnung", "gesamtmenge", "mainpath", "endpath"])
|
||||||
|
|
||||||
|
for tn in sorted(aggregated.keys()):
|
||||||
|
positions = [p for p in aggregated[tn]["positions"] if p]
|
||||||
|
mainpath = _common_path_prefix(positions)
|
||||||
|
variable_parts: list[str] = []
|
||||||
|
for pos in positions:
|
||||||
|
if mainpath:
|
||||||
|
suffix = pos[len(mainpath):].lstrip("/")
|
||||||
|
else:
|
||||||
|
suffix = pos
|
||||||
|
if suffix and suffix not in variable_parts:
|
||||||
|
variable_parts.append(suffix)
|
||||||
|
endpath = ",".join(variable_parts)
|
||||||
|
result_row = [
|
||||||
|
aggregated[tn]["teilenummer"],
|
||||||
|
aggregated[tn]["bezeichnung"],
|
||||||
|
aggregated[tn]["gesamtmenge"],
|
||||||
|
mainpath,
|
||||||
|
endpath,
|
||||||
|
]
|
||||||
|
ws_result.append(result_row)
|
||||||
|
if tn in product_numbers:
|
||||||
|
ws_product.append(result_row)
|
||||||
|
|
||||||
|
wb.save(str(xlsx_out))
|
||||||
|
wb.close()
|
||||||
|
log.info("XLSX geschrieben: %s", xlsx_out)
|
||||||
|
|
||||||
|
|
||||||
|
def _process_single_folder(
|
||||||
|
folder: Path,
|
||||||
|
example_dir: Path,
|
||||||
|
result: dict[str, list[dict[str, Any]]],
|
||||||
|
log: logging.Logger,
|
||||||
|
write_json: bool = False,
|
||||||
|
write_csv: bool = False,
|
||||||
|
write_xlsx: bool = False,
|
||||||
|
) -> None:
|
||||||
xlsx_files = sorted(folder.glob("*.xlsx"))
|
xlsx_files = sorted(folder.glob("*.xlsx"))
|
||||||
if not xlsx_files:
|
if not xlsx_files:
|
||||||
return
|
return
|
||||||
@@ -92,17 +297,34 @@ def _process_single_folder(folder: Path, example_dir: Path, result: dict, log) -
|
|||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
log.warning("%s/%s uebersprungen: %s", rel_folder, xlsx_path.name, exc)
|
log.warning("%s/%s uebersprungen: %s", rel_folder, xlsx_path.name, exc)
|
||||||
return
|
return
|
||||||
|
size_path = (3, 4)
|
||||||
|
|
||||||
result[rel_folder] = [
|
filtered_items = [
|
||||||
{
|
{
|
||||||
"position": item.position,
|
"position": item.position,
|
||||||
"teilenummer": item.teilenummer,
|
"teilenummer": item.teilenummer,
|
||||||
|
"bezeichnung": item.bezeichnung,
|
||||||
"menge": item.menge,
|
"menge": item.menge,
|
||||||
}
|
}
|
||||||
for item in items
|
for item in items
|
||||||
|
if item.position.count("/") in size_path
|
||||||
]
|
]
|
||||||
|
result[rel_folder] = filtered_items
|
||||||
|
write_results(
|
||||||
|
xlsx_path,
|
||||||
|
filtered_items,
|
||||||
|
log,
|
||||||
|
write_json=write_json,
|
||||||
|
write_csv=write_csv,
|
||||||
|
write_xlsx=write_xlsx,
|
||||||
|
)
|
||||||
|
|
||||||
def process_all(example_dir: Path) -> dict[str, list[dict[str, Any]]]:
|
def process_all(
|
||||||
|
example_dir: Path,
|
||||||
|
write_json: bool = False,
|
||||||
|
write_csv: bool = False,
|
||||||
|
write_xlsx: bool = False,
|
||||||
|
) -> dict[str, list[dict[str, Any]]]:
|
||||||
"""
|
"""
|
||||||
Durchsucht rekursiv alle Unterverzeichnisse in example_dir.
|
Durchsucht rekursiv alle Unterverzeichnisse in example_dir.
|
||||||
|
|
||||||
@@ -123,7 +345,15 @@ def process_all(example_dir: Path) -> dict[str, list[dict[str, Any]]]:
|
|||||||
result: dict[str, list[dict[str, Any]]] = {}
|
result: dict[str, list[dict[str, Any]]] = {}
|
||||||
for folder in subdirs:
|
for folder in subdirs:
|
||||||
log.info("Verarbeite %s", folder)
|
log.info("Verarbeite %s", folder)
|
||||||
_process_single_folder(folder, example_dir, result, log)
|
_process_single_folder(
|
||||||
|
folder,
|
||||||
|
example_dir,
|
||||||
|
result,
|
||||||
|
log,
|
||||||
|
write_json=write_json,
|
||||||
|
write_csv=write_csv,
|
||||||
|
write_xlsx=write_xlsx,
|
||||||
|
)
|
||||||
|
|
||||||
log.info("Fertig. Gelesene Verzeichnisse mit Daten: %d", len(result))
|
log.info("Fertig. Gelesene Verzeichnisse mit Daten: %d", len(result))
|
||||||
return result
|
return result
|
||||||
@@ -144,6 +374,24 @@ def main() -> None:
|
|||||||
)
|
)
|
||||||
parser.add_argument("--verbose", "-v", action="store_true",
|
parser.add_argument("--verbose", "-v", action="store_true",
|
||||||
help="Ausfuehrliche Ausgabe")
|
help="Ausfuehrliche Ausgabe")
|
||||||
|
parser.add_argument(
|
||||||
|
"--2json",
|
||||||
|
dest="to_json",
|
||||||
|
action="store_true",
|
||||||
|
help="Schreibt Ausgabe als .json neben die jeweilige .xlsx",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--2csv",
|
||||||
|
dest="to_csv",
|
||||||
|
action="store_true",
|
||||||
|
help="Schreibt Ausgabe als .csv neben die jeweilige .xlsx",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--2xlsx",
|
||||||
|
dest="to_xlsx",
|
||||||
|
action="store_true",
|
||||||
|
help="Schreibt Ausgabe als _result.xlsx neben die jeweilige .xlsx",
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
@@ -165,8 +413,14 @@ def main() -> None:
|
|||||||
|
|
||||||
if args.example_dir is None:
|
if args.example_dir is None:
|
||||||
log.info("Beispiel-Verzeichnis (alle Unterordner): %s", examples_root)
|
log.info("Beispiel-Verzeichnis (alle Unterordner): %s", examples_root)
|
||||||
data = process_all(examples_root)
|
data = process_all(
|
||||||
|
examples_root,
|
||||||
|
write_json=args.to_json,
|
||||||
|
write_csv=args.to_csv,
|
||||||
|
write_xlsx=args.to_xlsx,
|
||||||
|
)
|
||||||
log.info("Objekt erzeugt mit %d Schluesseln", len(data))
|
log.info("Objekt erzeugt mit %d Schluesseln", len(data))
|
||||||
|
print(json.dumps(data, indent=2, ensure_ascii=False, default=str))
|
||||||
return
|
return
|
||||||
|
|
||||||
# --example-dir: entweder direkter Verzeichnispfad oder Unterordnername in PV_EXAMPLES
|
# --example-dir: entweder direkter Verzeichnispfad oder Unterordnername in PV_EXAMPLES
|
||||||
@@ -189,8 +443,17 @@ def main() -> None:
|
|||||||
|
|
||||||
log.info("Beispiel-Verzeichnis (einzeln): %s", target_dir)
|
log.info("Beispiel-Verzeichnis (einzeln): %s", target_dir)
|
||||||
result: dict[str, list[dict[str, Any]]] = {}
|
result: dict[str, list[dict[str, Any]]] = {}
|
||||||
_process_single_folder(target_dir, target_dir.parent, result, log)
|
_process_single_folder(
|
||||||
|
target_dir,
|
||||||
|
target_dir.parent,
|
||||||
|
result,
|
||||||
|
log,
|
||||||
|
write_json=args.to_json,
|
||||||
|
write_csv=args.to_csv,
|
||||||
|
write_xlsx=args.to_xlsx,
|
||||||
|
)
|
||||||
log.info("Objekt erzeugt mit %d Schluesseln", len(result))
|
log.info("Objekt erzeugt mit %d Schluesseln", len(result))
|
||||||
|
#print(json.dumps(result, indent=2, ensure_ascii=False, default=str))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
+58
-28
@@ -81,19 +81,17 @@ def _extract_toc_numbers(doc) -> dict[str, str]:
|
|||||||
"""Extrahiert Kapitelnummern aus dem Inhaltsverzeichnis des PDFs.
|
"""Extrahiert Kapitelnummern aus dem Inhaltsverzeichnis des PDFs.
|
||||||
|
|
||||||
Sucht TOC-Seiten (Seiten mit '....' Punkt-Fuellern) und ordnet
|
Sucht TOC-Seiten (Seiten mit '....' Punkt-Fuellern) und ordnet
|
||||||
Ueberschriftentexte ihren Kapitelnummern zu.
|
Ueberschriftentexte ihren Kapitelnummern zu. Nur Eintraege mit
|
||||||
|
expliziter Nummer werden erfasst (bis Level 2).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dict title_text -> kapitelnummer, z.B. {"Typen von Weichen mit Bögen": "2.3.1"}
|
Dict title_text -> kapitelnummer, z.B. {"Weichen Omniflo": "2.3"}
|
||||||
"""
|
"""
|
||||||
import fitz
|
|
||||||
|
|
||||||
toc_entries: dict[str, str] = {}
|
toc_entries: dict[str, str] = {}
|
||||||
|
|
||||||
for page_num in range(len(doc)):
|
for page_num in range(len(doc)):
|
||||||
page = doc[page_num]
|
page = doc[page_num]
|
||||||
text = page.get_text()
|
text = page.get_text()
|
||||||
# Nur TOC-Seiten verarbeiten (enthalten "....")
|
|
||||||
if "...." not in text:
|
if "...." not in text:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -106,8 +104,7 @@ def _extract_toc_numbers(doc) -> dict[str, str]:
|
|||||||
if not line:
|
if not line:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# TOC-Zeile: "Text ........... Seitennr"
|
# Kapitelnummer auf separater Zeile
|
||||||
# Kapitelnummer steht oft auf separater Zeile davor
|
|
||||||
num_match = re.match(r"^(\d+(?:\.\d+)*)\s*$", line)
|
num_match = re.match(r"^(\d+(?:\.\d+)*)\s*$", line)
|
||||||
if num_match:
|
if num_match:
|
||||||
current_number = num_match.group(1)
|
current_number = num_match.group(1)
|
||||||
@@ -115,32 +112,59 @@ def _extract_toc_numbers(doc) -> dict[str, str]:
|
|||||||
|
|
||||||
# Zeile mit Punkt-Fuellern = TOC-Eintrag
|
# Zeile mit Punkt-Fuellern = TOC-Eintrag
|
||||||
dot_match = re.search(r"\.{4,}", line)
|
dot_match = re.search(r"\.{4,}", line)
|
||||||
if dot_match:
|
if not dot_match:
|
||||||
# Text vor den Punkten extrahieren
|
continue
|
||||||
title = line[:dot_match.start()].strip()
|
|
||||||
# Seitennummer nach den Punkten
|
|
||||||
if title and current_number:
|
|
||||||
toc_entries[title] = current_number
|
|
||||||
current_number = ""
|
|
||||||
elif title:
|
|
||||||
# Nummer koennte am Anfang des Titels stehen
|
|
||||||
nm = re.match(r"^(\d+(?:\.\d+)*)\s+(.+)", title)
|
|
||||||
if nm:
|
|
||||||
toc_entries[nm.group(2).strip()] = nm.group(1)
|
|
||||||
|
|
||||||
# Auch inline Nummern: "1.4 Abkürzungen ..... 8"
|
title = line[:dot_match.start()].strip()
|
||||||
for line in lines:
|
if not title:
|
||||||
line = line.strip()
|
continue
|
||||||
dot_match = re.search(r"\.{4,}", line)
|
|
||||||
if dot_match:
|
# Inline-Nummer am Anfang? "1.4 Abkürzungen ....."
|
||||||
title_part = line[:dot_match.start()].strip()
|
nm = re.match(r"^(\d+(?:\.\d+)*)\s+(.+)", title)
|
||||||
nm = re.match(r"^(\d+(?:\.\d+)*)\s+(.+)", title_part)
|
if nm:
|
||||||
if nm:
|
toc_entries[nm.group(2).strip()] = nm.group(1)
|
||||||
toc_entries[nm.group(2).strip()] = nm.group(1)
|
current_number = ""
|
||||||
|
elif current_number:
|
||||||
|
toc_entries[title] = current_number
|
||||||
|
current_number = ""
|
||||||
|
|
||||||
return toc_entries
|
return toc_entries
|
||||||
|
|
||||||
|
|
||||||
|
RE_DATE = re.compile(r"^\d{2}\.\d{2}\.\d{4}$")
|
||||||
|
RE_COPYRIGHT = re.compile(r"^Copyright\s+\d{4}", re.IGNORECASE)
|
||||||
|
RE_VERSION_LINE = re.compile(r"^Version\s*[:.]?\s*\d", re.IGNORECASE)
|
||||||
|
|
||||||
|
# Bekannte Rausch-Muster: Autorinnen, Metadaten, Copyright-Zeilen
|
||||||
|
_NOISE_KEYWORDS = {
|
||||||
|
"annette schopper", "erstellt:", "freigabe:", "nicht freigegeben",
|
||||||
|
"nur zur information", "schönenberger systeme gmbh",
|
||||||
|
"justus-von-liebig-str", "info@schoenenberger.de",
|
||||||
|
"www.schoenenberger.de",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _is_noise(text: str) -> bool:
|
||||||
|
"""Erkennt Rausch-Zeilen die nicht ins RST gehoeren."""
|
||||||
|
lower = text.lower().strip()
|
||||||
|
if RE_COPYRIGHT.match(text):
|
||||||
|
return True
|
||||||
|
if RE_DATE.match(text.strip()):
|
||||||
|
return True
|
||||||
|
if RE_VERSION_LINE.match(text):
|
||||||
|
return True
|
||||||
|
for kw in _NOISE_KEYWORDS:
|
||||||
|
if kw in lower:
|
||||||
|
return True
|
||||||
|
# Seitenzahlen
|
||||||
|
if re.match(r"^Seite\s+\d+$", text, re.IGNORECASE):
|
||||||
|
return True
|
||||||
|
# Einzelne kurze Jahreszahlen mit Autorin: "2020 Annette Schopper"
|
||||||
|
if re.match(r"^\d{4}\s+[A-Z][a-z]+\s+[A-Z][a-z]+$", text):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def extract_text_to_rst(pdf_path: Path, rst_path: Path) -> int:
|
def extract_text_to_rst(pdf_path: Path, rst_path: Path) -> int:
|
||||||
"""Extrahiert Text aus PDF und schreibt ihn als .rst.
|
"""Extrahiert Text aus PDF und schreibt ihn als .rst.
|
||||||
|
|
||||||
@@ -188,6 +212,8 @@ def extract_text_to_rst(pdf_path: Path, rst_path: Path) -> int:
|
|||||||
text = text.strip()
|
text = text.strip()
|
||||||
if not text:
|
if not text:
|
||||||
continue
|
continue
|
||||||
|
if _is_noise(text):
|
||||||
|
continue
|
||||||
|
|
||||||
raw_lines.append({
|
raw_lines.append({
|
||||||
"text": text,
|
"text": text,
|
||||||
@@ -311,6 +337,10 @@ def extract_text_to_rst(pdf_path: Path, rst_path: Path) -> int:
|
|||||||
# Mehrfache Leerzeilen reduzieren
|
# Mehrfache Leerzeilen reduzieren
|
||||||
rst = "\n".join(rst_lines)
|
rst = "\n".join(rst_lines)
|
||||||
rst = re.sub(r"\n{3,}", "\n\n", rst)
|
rst = re.sub(r"\n{3,}", "\n\n", rst)
|
||||||
|
# 9-stellige Nummern mit Leerzeichen zusammenfuegen: "834 372 007" -> "834372007"
|
||||||
|
rst = re.sub(r"\b(\d{3})\s(\d{3})\s(\d{3})\b", r"\1\2\3", rst)
|
||||||
|
rst = re.sub(r"\b(\d{3})\s(\d{6})\b", r"\1\2", rst)
|
||||||
|
rst = re.sub(r"\b(\d{6})\s(\d{3})\b", r"\1\2", rst)
|
||||||
rst = rst.strip() + "\n"
|
rst = rst.strip() + "\n"
|
||||||
|
|
||||||
rst_path.parent.mkdir(parents=True, exist_ok=True)
|
rst_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|||||||
Reference in New Issue
Block a user