diff --git a/bin/create_examples.bat b/bin/create_examples.bat new file mode 100644 index 0000000..dc02733 --- /dev/null +++ b/bin/create_examples.bat @@ -0,0 +1,20 @@ +@echo off +REM ================================================================ +REM DOCU_BUILD - Bilder und Text aus Produktprogramm-PDFs extrahieren +REM ================================================================ + +call "%~dp0setenv.bat" + +if not exist "%PROJECT%\.venv" ( + echo FEHLER: Virtual environment nicht gefunden. + echo Bitte zuerst bin\install_py.bat ausfuehren. + exit /b 1 +) + +call "%PROJECT%\.venv\Scripts\activate.bat" + +echo. +echo Extrahiere Bilder und Text aus Produktprogramm-PDFs ... +echo. + +python "%PV_LIB%\create_examples.py" --example-dir fortna %* diff --git a/cfg/product_numbers.cfg b/cfg/product_numbers.cfg new file mode 100644 index 0000000..33777e5 --- /dev/null +++ b/cfg/product_numbers.cfg @@ -0,0 +1,1419 @@ +# Teilenummern aus Produktprogramm-Dokumenten +# Extrahiert aus data/Produktprogramm/*.de.rst +# Format: eine 9-stellige Teilenummer pro Zeile + +[Omniflo] +200000189 +400001435 +400100480 +400101281 +400101282 +400101708 +400101742 +400101820 +400101865 +400102101 +704003041 +710005021 +720000007 +720000008 +723000000 +790000045 +790000203 +790823001 +821004071 +821006004 +821006101 +821064080 +821084000 +821084200 +821084201 +821086018 +821094040 +821094101 +821096001 +821104017 +821104019 +821104021 +821104022 +821104024 +821104025 +821104026 +821104027 +821104028 +821104029 +821104030 +821104031 +821104033 +821104034 +821104035 +821104037 +821104038 +821104039 +821104041 +821104043 +821104065 +821104066 +821106001 +821106002 +821106110 +821114001 +821114002 +821114004 +821114006 +821114010 +821114011 +821114015 +821114016 +821114017 +821114034 +821114040 +821114075 +821116001 +821116017 +821116062 +821116078 +821116079 +821126002 +821126003 +821204042 +821204043 +821204045 +821204100 +821214100 +821332001 +821332002 +821332003 +821332004 +821332005 +821332006 +822004001 +822012620 +822012621 +822012622 +822012623 +822012624 +822012625 +822012626 +822012628 +822012629 +822026004 +822034002 +822034003 +822034004 +822034009 +822034010 +822034015 +822034016 +822034025 +822034052 +822035037 +822035100 +822035102 +822035103 +822035206 +822037145 +822062007 +822062008 +822064013 +822064035 +822064071 +822064072 +822064075 +822064076 +822066026 +822066032 +822066033 +822066034 +822066035 +822066087 +822066088 +822066127 +822066128 +822066163 +822066164 +822066165 +822066166 +822066168 +822066218 +822066219 +822122004 +822132009 +822132010 +822132012 +822134038 +822134043 +822134044 +822134047 +822136052 +822136088 +822136168 +822136175 +822136184 +822136194 +822136200 +822136206 +824111001 +824111002 +824111004 +824111005 +824111010 +824111102 +824111103 +824111104 +824111105 +824111106 +824111107 +824111110 +824111111 +824144025 +824312014 +824312016 +824312017 +824331125 +824602001 +824604001 +824604002 +824604008 +824604009 +824604011 +824604012 +824604030 +824604031 +824604032 +824604033 +825002016 +825004005 +825004006 +825004017 +825004034 +825004052 +826004100 +827024017 +827062002 +827062022 +827062028 +827062038 +827062101 +827062102 +827062103 +827062104 +827064039 +827064040 +827064041 +827064044 +827072013 +827072014 +827072015 +827072016 +827072100 +827072103 +827072200 +827072201 +827072202 +827072203 +827072204 +827072205 +827072207 +827072208 +827074011 +827092004 +827092005 +827334022 +827902201 +827902202 +827902211 +827902212 +827902213 +827902216 +827902218 +827902220 +827902222 +827902223 +827904201 +828004005 +828004010 +831214037 +831214038 +831214039 +831214040 +831214041 +831214042 +831214043 +831214044 +831214045 +831216022 +831216024 +834244662 +834244663 +834244666 +834342001 +834342002 +834342003 +834342004 +834342005 +834342011 +834342012 +834342013 +834342014 +834342015 +834342100 +834342101 +834342102 +834342103 +834342200 +834342201 +834342202 +834372001 +834372002 +834372003 +834372004 +834372005 +834372007 +834372008 +834372009 +834372010 +834372011 +834372021 +834372022 +834372023 +834372024 +834372025 +834372027 +834372028 +834372029 +834372030 +834372031 +834372047 +834372048 +834372049 +834372050 +834372051 +834372100 +834372101 +834372102 +834372103 +834372104 +834372105 +834372106 +834372107 +834372108 +834372109 +834372110 +834372111 +834372115 +834372116 +834372117 +834372200 +834372201 +834372202 +834372203 +834372204 +834372205 +834372206 +834372207 +834372208 +834372209 +834372210 +834372211 +834372215 +834372216 +834372217 +834372400 +834372401 +834372402 +834372403 +834372404 +834372405 +834372420 +834372500 +834372501 +834372502 +834372503 +834372504 +834372512 +834372513 +834372514 +834372515 +834372516 +834372524 +834372525 +834372526 +834372527 +834372528 +834372536 +834372537 +834372538 +834372539 +834372540 +834372563 +834372566 +834372600 +834372601 +834372602 +834372610 +834372611 +834372612 +834372613 +834372614 +834372620 +834372621 +834372622 +834372623 +834372624 +888006086 +899500001 +899500002 +899500003 +911021005 +911021105 +911021117 +911021172 +911021254 +911091037 +911261015 +911261023 +914062007 +916051001 +919010054 +919050001 +919050002 +922081020 +924021044 +924031009 +924031058 +924031081 +924031093 +929041020 +929041021 +929041030 +929055161 +929055170 +929055175 +929055185 +929055247 +929055259 +929055260 +929055269 +929055270 +929055274 +929055279 +929055282 +929055283 +929055284 +929055287 +929055288 +929055289 +929055290 +929055293 +929055294 +929055382 +929055383 +929055384 +929055387 +929055388 +929055389 +929120030 +929120031 +929120032 +929120101 +929120102 +931041013 + +[ILS] +200000016 +200000017 +200000018 +200000019 +200000020 +200000021 +200000040 +200000071 +200000077 +200000086 +200000146 +200000169 +200000217 +200000241 +200000252 +200000253 +400000956 +400100789 +400100790 +400102122 +400102190 +400102274 +400102318 +400102325 +400102388 +400102478 +400102632 +400102656 +720002003 +821004058 +821004112 +821006038 +821006040 +821404001 +821404002 +821404003 +821404004 +821404005 +821406010 +821406020 +821416006 +825026010 +825026011 +825026012 +825026013 +825026014 +827334016 +827422001 +827422002 +827422003 +827422004 +829019020 +829412009 +829412010 +829412011 +829412012 +829412013 +829412015 +829412016 +829412017 +829412021 +829412050 +829412051 +829412052 +829412053 +829414001 +829414002 +829414003 +829414004 +829414005 +829414006 +829414013 +829414014 +829414200 +829414230 +829414231 +829416030 +829416031 +829416032 +829416065 +829416067 +829416068 +829416073 +829416105 +829416120 +829416121 +829416122 +829416124 +829416130 +829416131 +829416220 +829416319 +829416323 +829416324 +829416325 +829422025 +829422026 +829424400 +829426402 +829426403 +829434300 +829434301 +829434302 +829434303 +829434304 +829434305 +829434307 +829434308 +829434313 +829434314 +829434322 +829434329 +829434330 +829436011 +829436022 +829436023 +829436024 +829436025 +829436026 +829436027 +829442001 +829442005 +829442006 +829442100 +829442101 +829442103 +829442601 +829442602 +829444020 +829444025 +829452301 +829452302 +829452303 +829452304 +829452401 +829452402 +829452403 +829452404 +829454322 +829454323 +829454451 +829454452 +829454453 +829454454 +829454455 +829454456 +829454457 +829454458 +829454459 +829454460 +829454461 +829456306 +829456452 +829456453 +829456900 +829462001 +829462002 +829462003 +829462011 +829514001 +829514002 +829514003 +829514004 +829514005 +829514061 +829514062 +829514064 +829514065 +829514066 +829514071 +829514073 +829514074 +829514075 +829514076 +829514077 +829514078 +829514080 +829514081 +829514082 +829514084 +829514085 +829514086 +829514221 +829516001 +829516002 +829516003 +829516004 +829516005 +829516010 +829516016 +829516023 +829516025 +829516026 +829516027 +829516028 +829516029 +829516030 +829516031 +829516034 +829516035 +829516036 +829516041 +829516042 +829516043 +829516044 +829516045 +829516047 +829516048 +829516049 +829516050 +829516051 +829516052 +829516053 +829516054 +829516055 +829516056 +829516072 +829516201 +829522001 +829522002 +829522003 +829522021 +829522022 +829522023 +829522024 +829522502 +829522503 +829522506 +829522507 +829522510 +829522511 +829524055 +829524101 +829524102 +829524103 +829524104 +829524105 +829524106 +829524107 +829524108 +829524109 +829524110 +829524111 +829524112 +829524501 +829524505 +829524509 +829524512 +829524513 +829524514 +829524515 +829524602 +829524603 +829524604 +829524605 +829524606 +829524607 +829526000 +829526003 +829532001 +829532002 +829532003 +829532004 +829532005 +829532006 +829532007 +829532009 +829532010 +829532011 +829532012 +829532013 +829532014 +829532015 +829532016 +829532601 +829532602 +829532603 +829532604 +829534002 +829534004 +829534005 +829534006 +829534007 +829534008 +829534016 +829534019 +829534022 +829534023 +829534024 +829534025 +829534029 +829534030 +829534031 +829534032 +829534101 +829534102 +829534103 +829534130 +829534131 +829534132 +829534133 +829534301 +829534302 +829534303 +829534304 +829534305 +829536004 +829572001 +829572002 +829572003 +829572004 +829572005 +829572006 +829572011 +829572012 +829572013 +829572014 +829572015 +829572016 +829622024 +829622550 +829622551 +829622552 +829622553 +829662001 +829662003 +829662011 +829662012 +829664005 +829664006 +829664008 +911011013 +911011933 +911021012 +911021014 +911021019 +911021132 +911021232 +911071038 +911271050 +911271100 +911271571 +913011009 +913011016 +914011068 +914031009 +926025079 +929055244 +929055250 +929055251 +929055252 +929055254 +929055255 +929055256 +929055292 +931011103 +931601004 + +[CPC] +574002402 +610014001 +610014002 +610014003 +610014021 +610014022 +610014023 +610014031 +610014032 +610014033 +610054001 +610054002 +610102501 +610104001 +610104002 +610104101 +610154001 +610154002 +610154005 +610154101 +610154201 +610154202 +610154203 +610154204 +610154205 +610154206 +610154207 +610154250 +610154301 +610154302 +610154303 +610154304 +610154305 +610154306 +610154307 +610154350 +610156004 +610164001 +610164002 +610164003 +610164004 +610174001 +610204001 +610204002 +610204101 +610204102 +610204104 +610206002 +610224201 +610252003 +610254001 +610254002 +610254006 +610254007 +610254008 +610254009 +610254010 +610254011 +610254013 +610254014 +610254015 +610264001 +610264002 +610274001 +610274002 +610404001 +610404002 +610404003 +610404004 +610454001 +610454002 +610454003 +610454004 +610454005 +610454006 +610454007 +610454008 +610504001 +610504003 +610504004 +610504005 +610504010 +610506009 +610506013 +610506042 +610554001 +610554003 +610554004 +610554005 +610554006 +610554007 +610554008 +610554009 +610554010 +610804001 +610804101 +610804102 +610854001 +610854002 +610854003 +610854004 +821004133 +821004134 +821004135 +821004136 +821126001 +822074010 +822074014 +822074015 +822074025 +822074026 +822074027 +822074030 +822074031 +822074037 +822074038 +822074041 +822074042 +822074044 +822074046 +822074047 +822074048 +822074049 +822074050 +822074051 +822076044 +822076049 +822076103 +822174031 +822174032 +824704005 +824824001 +824824002 +824824003 +824824004 +824824008 +824824009 +824824011 +824824012 +824824026 +824824028 +824824032 +824824033 +824824034 +824824036 +824826024 +824826025 +824826026 +824826030 +824826031 +824826044 +825824001 +825824002 +825824004 +825824005 +825824006 +825824007 +825824008 +825824009 +825824010 +826604005 +826606013 +827024001 +827024002 +827072102 +827084001 +827174005 +827424005 +842191147 +842191200 +842501777 +911021077 +912992001 +913011016 +915031005 +916021081 +916021102 +924031039 +929012184 +929012489 +929012523 +929055205 +929055206 +929055298 +929055299 +929055303 +931041085 +931051035 +931501001 +931991002 +933021028 +933021030 + +[Trolley] +141810046 +400001413 +400001414 +400001415 +400001416 +400001417 +400001418 +400001419 +400001420 +400001421 +400001730 +400100138 +560103201 +569002201 +569002401 +569002402 +711001022 +711001023 +825004042 +826002201 +826002300 +826002301 +826002310 +826002321 +826002322 +826004013 +826004015 +826004016 +826004022 +826004033 +826004102 +826004150 +826004211 +826004212 +826004213 +826004215 +826004216 +826004230 +826004232 +826004237 +826004238 +826004239 +826004240 +826004301 +826004302 +826004303 +826004304 +826004306 +826004309 +826004311 +826004312 +826004321 +826004322 +826004323 +826004331 +826004332 +826004401 +826004402 +826004403 +826004404 +826004431 +826004432 +826004433 +826004434 +826004602 +826005020 +826005031 +826005032 +826005051 +826005055 +826005058 +826005062 +826005063 +826005065 +826005067 +826005068 +826005072 +826005073 +826005075 +826005083 +826005084 +826006034 +826006201 +826006202 +826006203 +826006215 +826022002 +826022005 +826022010 +826022013 +826022024 +826022042 +826022043 +826022044 +826022046 +826022047 +826022049 +826022050 +826022053 +826022064 +826022065 +826022066 +826022068 +826024006 +826026002 +826027023 +826027025 +826226075 +826226360 +826312001 +826312017 +826315022 +826315025 +826315028 +826315030 +826602002 +826802001 +826802002 +826802003 +826802004 +826802005 +826802006 +826802007 +826804001 +826812002 +826812004 +826812005 +826822001 +826822002 +826822003 +826822004 +826822005 +826822008 +826822009 +826822010 +826822011 +826822012 +826822013 +826822014 +826822015 +826822016 +826824003 +826824004 +826824005 +826824006 +826824009 +826824010 +826824011 +826824012 +826824013 +826824014 +826824015 +826824016 +826824017 +826824018 +826832004 +826832005 +826832006 +826842001 +826862001 +826864003 +826904033 +826904200 +826904202 +826904203 +826904204 +826904205 +826904206 +826904207 +826904208 +826904209 +826904210 +826904211 +826904212 +826904213 +826904214 +826904215 +826904220 +826906202 +826906204 +826907006 +826914001 +827902200 +828506002 +888006005 +914011030 +919010014 +922041044 +924045001 +924045034 +926025049 + +[Geruest] +821002010 +821002015 +821004003 +821004029 +821004038 +821004051 +821004052 +821004053 +821004054 +821004058 +821004059 +821004060 +821004112 +821004115 +821004116 +821004117 +821004118 +821004120 +821004121 +821004122 +821004123 +821004124 +821004125 +821004126 +821004127 +821004128 +821004129 +821004130 +821004131 +821004132 +821004133 +821004134 +821004135 +821004136 +821004137 +821004138 +821004139 +821004140 +821004160 +821004161 +821004162 +821004163 +821004164 +821004165 +821004166 +821004167 +821004168 +821004169 +821004170 +821004171 +821004172 +821004173 +821004174 +821004175 +821004176 +821004177 +821004178 +821004179 +821004180 +821004181 +821004182 +821004183 +821004184 +821004185 +821004186 +821004187 +821004188 +821004189 +821004190 +821004191 +821004192 +821004193 +821004194 +821004195 +821004196 +821004197 +821004198 +821004199 +821004200 +821004220 +821004221 +821004222 +821004223 +821004224 +821004225 +821004226 +821004227 +821004228 +821004229 +821004230 +821004231 +821004232 +821004233 +821004234 +821004235 +821004236 +821004237 +821004238 +821004239 +821004240 +821004270 +821004271 +821004272 +821004273 +821004274 +821004275 +821004276 +821004277 +821004278 +821004279 +821004280 +821004281 +821004282 +821004283 +821004284 +821004285 +821004286 +821004287 +821004288 +821004289 +821004290 +821004291 +821004292 +821004293 +821004294 +821004295 +821004296 +821004297 +821004298 +821004299 +821004300 +821006001 +821006005 +821006006 +821006009 +821006021 +821006024 +821006026 +821006027 +821006031 +821006040 +821006041 +821006042 +821006043 +821006120 +821006300 +821114009 +821114013 +821114018 +821114019 +821114025 +821114027 +821114028 +821114029 +821114036 +821114061 +821114076 +821114080 +821114081 +821114084 +821114085 +821116011 +821116014 +821116025 +821116026 +821116027 +821116028 +825002007 +825002008 +825002009 +825002019 +825002020 +825002021 +825002022 +825004006 +825004020 +825004022 +825004027 +825004028 +825004030 +825004040 +825004046 +825004047 +825004048 +825004049 +825004050 +825006006 +825006009 +825022075 +825026009 +825026010 +825026011 +825026012 +825026013 +825026014 +825084001 +825084002 +825084003 +825084004 +825084005 +825084006 +825104001 +825104002 +825104003 +825104004 +825104005 +825104006 +825144001 +825144002 +825144003 +825144004 +825144005 +825144006 +831214101 +831314101 +911011004 +911011008 +911011009 +911011070 +911011121 +911021143 +911021912 +911071006 +911121039 +911271027 +912011009 +912011010 +912041005 +912041985 +913011009 +913011010 +913011021 +913011140 +913031009 +913031010 +920020012 +922013003 +922013004 +922021003 +922021004 +922021011 +922021012 +922081021 +922081028 +922081029 +922081030 +922083009 +924021034 +924021041 +926013011 diff --git a/examples/Fortna/KA135776 FortnaHM20260416 NST.xlsx b/examples/Fortna/KA135776 FortnaHM20260416 NST.xlsx new file mode 100644 index 0000000..ccb862b Binary files /dev/null and b/examples/Fortna/KA135776 FortnaHM20260416 NST.xlsx differ diff --git a/lib/create_examples.py b/lib/create_examples.py index bba15c8..de30b8e 100644 --- a/lib/create_examples.py +++ b/lib/create_examples.py @@ -8,8 +8,12 @@ Verwendung: from __future__ import annotations import argparse +import csv +import json import logging +import re import sys +from collections import defaultdict from dataclasses import dataclass from pathlib import Path from typing import Any @@ -21,6 +25,7 @@ log = logging.getLogger(__name__) class StkItem: position: str teilenummer: str + bezeichnung: str menge: Any @@ -47,7 +52,7 @@ def _extract_stk_liste_ist(xlsx_path: Path) -> list[StkItem]: ws = wb["Stk.-Liste IST"] - header_idx: tuple[int | None, int | None, int | None] | None = None + header_idx: tuple[int | None, int | None, int | None, int | None] | None = None items: list[StkItem] = [] for row in ws.iter_rows(values_only=True): @@ -57,30 +62,230 @@ def _extract_stk_liste_ist(xlsx_path: Path) -> list[StkItem]: list(row), ["teilenummer", "teilnummer", "article", "artikelnummer"], ) + desc_idx = _find_col_index( + list(row), + ["bezeichnung", "beschreibung", "description", "benennung"], + ) qty_idx = _find_col_index(list(row), ["menge", "anzahl", "qty", "quantity"]) - if pos_idx is not None and part_idx is not None and qty_idx is not None: - header_idx = (pos_idx, part_idx, qty_idx) + if ( + pos_idx is not None + and part_idx is not None + and desc_idx is not None + and qty_idx is not None + ): + header_idx = (pos_idx, part_idx, desc_idx, qty_idx) continue - pos_idx, part_idx, qty_idx = header_idx - assert pos_idx is not None and part_idx is not None and qty_idx is not None + pos_idx, part_idx, desc_idx, qty_idx = header_idx + assert ( + pos_idx is not None + and part_idx is not None + and desc_idx is not None + and qty_idx is not None + ) position = _cell_str(row[pos_idx]) if pos_idx < len(row) else "" teilenummer = _cell_str(row[part_idx]) if part_idx < len(row) else "" + bezeichnung = _cell_str(row[desc_idx]) if desc_idx < len(row) else "" menge = row[qty_idx] if qty_idx < len(row) else None # Leere Zeilen ueberspringen - if not position and not teilenummer and menge in ("", None): + if not position and not teilenummer and not bezeichnung and menge in ("", None): continue if not position or not teilenummer: continue - items.append(StkItem(position=position, teilenummer=teilenummer, menge=menge)) + items.append( + StkItem( + position=position, + teilenummer=teilenummer, + bezeichnung=bezeichnung, + menge=menge, + ) + ) wb.close() return items -def _process_single_folder(folder: Path, example_dir: Path, result: dict, log) -> None: + +def _to_number(value: Any) -> float: + if value is None: + return 0.0 + if isinstance(value, (int, float)): + return float(value) + text = str(value).strip().replace(",", ".") + if not text: + return 0.0 + try: + return float(text) + except ValueError: + return 0.0 + + +def _common_path_prefix(paths: list[str]) -> str: + parts_list = [p.split("/") for p in paths if p] + if not parts_list: + return "" + common = parts_list[0] + for parts in parts_list[1:]: + i = 0 + max_i = min(len(common), len(parts)) + while i < max_i and common[i] == parts[i]: + i += 1 + common = common[:i] + if not common: + break + return "/".join(common) + + +def _load_product_numbers(cfg_path: Path) -> dict[str, str]: + """ + Laedt 9-stellige Produktnummern aus cfg/product_numbers.cfg. + + Erwartetes Format: + [Sektion] + 123456789 + 987654321 + """ + allowed_sections = {"omniflo", "ils", "cpc", "trolley", "geruest"} + products: dict[str, str] = {} + current_section = "" + + if not cfg_path.exists(): + return products + + for raw_line in cfg_path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + + if line.startswith("[") and line.endswith("]"): + current_section = line[1:-1].strip().lower() + continue + + if current_section not in allowed_sections: + continue + + if re.match(r"^\d{9}$", line): + products[line] = current_section + + return products + + +def write_results( + xlsx_path: Path, + filtered_items: list[dict[str, Any]], + log: logging.Logger, + write_json: bool = False, + write_csv: bool = False, + write_xlsx: bool = False, +) -> None: + if write_json: + json_path = xlsx_path.with_suffix(".json") + with json_path.open("w", encoding="utf-8") as fh: + json.dump(filtered_items, fh, indent=2, ensure_ascii=False, default=str) + log.info("JSON geschrieben: %s", json_path) + + if write_csv: + csv_path = xlsx_path.with_suffix(".csv") + with csv_path.open("w", encoding="utf-8-sig", newline="") as fh: + writer = csv.writer(fh, delimiter=";") + writer.writerow(["position", "teilenummer", "bezeichnung", "menge"]) + for row in filtered_items: + writer.writerow( + [ + row.get("position", ""), + row.get("teilenummer", ""), + row.get("bezeichnung", ""), + row.get("menge", ""), + ] + ) + log.info("CSV geschrieben: %s", csv_path) + + if write_xlsx: + import openpyxl + + xlsx_out = xlsx_path.with_name(f"{xlsx_path.stem}_result.xlsx") + wb = openpyxl.Workbook() + ws_3 = wb.active + ws_3.title = "3er" + ws_4 = wb.create_sheet("4er") + ws_result = wb.create_sheet("Result") + ws_product = wb.create_sheet("product") + + headers = ["position", "teilenummer", "bezeichnung", "menge"] + ws_3.append(headers) + ws_4.append(headers) + + aggregated: dict[str, dict[str, Any]] = defaultdict( + lambda: { + "teilenummer": "", + "bezeichnung": "", + "gesamtmenge": 0.0, + "positions": [], + } + ) + + for row in filtered_items: + slash_count = str(row.get("position", "")).count("/") + if slash_count == 3: + ws_3.append([row.get(h, "") for h in headers]) + elif slash_count == 4: + ws_4.append([row.get(h, "") for h in headers]) + + tn = str(row.get("teilenummer", "")).strip() + if not tn: + continue + if not aggregated[tn]["teilenummer"]: + aggregated[tn]["teilenummer"] = tn + aggregated[tn]["bezeichnung"] = row.get("bezeichnung", "") + aggregated[tn]["gesamtmenge"] += _to_number(row.get("menge")) + aggregated[tn]["positions"].append(str(row.get("position", "")).strip()) + + ws_result.append(["teilenummer", "bezeichnung", "gesamtmenge", "mainpath", "endpath"]) + + project_root = Path(__file__).resolve().parent.parent + product_cfg = project_root / "cfg" / "product_numbers.cfg" + product_numbers = _load_product_numbers(product_cfg) + ws_product.append(["teilenummer", "bezeichnung", "gesamtmenge", "mainpath", "endpath"]) + + for tn in sorted(aggregated.keys()): + positions = [p for p in aggregated[tn]["positions"] if p] + mainpath = _common_path_prefix(positions) + variable_parts: list[str] = [] + for pos in positions: + if mainpath: + suffix = pos[len(mainpath):].lstrip("/") + else: + suffix = pos + if suffix and suffix not in variable_parts: + variable_parts.append(suffix) + endpath = ",".join(variable_parts) + result_row = [ + aggregated[tn]["teilenummer"], + aggregated[tn]["bezeichnung"], + aggregated[tn]["gesamtmenge"], + mainpath, + endpath, + ] + ws_result.append(result_row) + if tn in product_numbers: + ws_product.append(result_row) + + wb.save(str(xlsx_out)) + wb.close() + log.info("XLSX geschrieben: %s", xlsx_out) + + +def _process_single_folder( + folder: Path, + example_dir: Path, + result: dict[str, list[dict[str, Any]]], + log: logging.Logger, + write_json: bool = False, + write_csv: bool = False, + write_xlsx: bool = False, +) -> None: xlsx_files = sorted(folder.glob("*.xlsx")) if not xlsx_files: return @@ -92,17 +297,34 @@ def _process_single_folder(folder: Path, example_dir: Path, result: dict, log) - except ValueError as exc: log.warning("%s/%s uebersprungen: %s", rel_folder, xlsx_path.name, exc) return + size_path = (3, 4) - result[rel_folder] = [ + filtered_items = [ { "position": item.position, "teilenummer": item.teilenummer, + "bezeichnung": item.bezeichnung, "menge": item.menge, } for item in items + if item.position.count("/") in size_path ] + result[rel_folder] = filtered_items + write_results( + xlsx_path, + filtered_items, + log, + write_json=write_json, + write_csv=write_csv, + write_xlsx=write_xlsx, + ) -def process_all(example_dir: Path) -> dict[str, list[dict[str, Any]]]: +def process_all( + example_dir: Path, + write_json: bool = False, + write_csv: bool = False, + write_xlsx: bool = False, +) -> dict[str, list[dict[str, Any]]]: """ Durchsucht rekursiv alle Unterverzeichnisse in example_dir. @@ -123,7 +345,15 @@ def process_all(example_dir: Path) -> dict[str, list[dict[str, Any]]]: result: dict[str, list[dict[str, Any]]] = {} for folder in subdirs: log.info("Verarbeite %s", folder) - _process_single_folder(folder, example_dir, result, log) + _process_single_folder( + folder, + example_dir, + result, + log, + write_json=write_json, + write_csv=write_csv, + write_xlsx=write_xlsx, + ) log.info("Fertig. Gelesene Verzeichnisse mit Daten: %d", len(result)) return result @@ -144,6 +374,24 @@ def main() -> None: ) parser.add_argument("--verbose", "-v", action="store_true", help="Ausfuehrliche Ausgabe") + parser.add_argument( + "--2json", + dest="to_json", + action="store_true", + help="Schreibt Ausgabe als .json neben die jeweilige .xlsx", + ) + parser.add_argument( + "--2csv", + dest="to_csv", + action="store_true", + help="Schreibt Ausgabe als .csv neben die jeweilige .xlsx", + ) + parser.add_argument( + "--2xlsx", + dest="to_xlsx", + action="store_true", + help="Schreibt Ausgabe als _result.xlsx neben die jeweilige .xlsx", + ) args = parser.parse_args() logging.basicConfig( @@ -165,8 +413,14 @@ def main() -> None: if args.example_dir is None: log.info("Beispiel-Verzeichnis (alle Unterordner): %s", examples_root) - data = process_all(examples_root) + data = process_all( + examples_root, + write_json=args.to_json, + write_csv=args.to_csv, + write_xlsx=args.to_xlsx, + ) log.info("Objekt erzeugt mit %d Schluesseln", len(data)) + print(json.dumps(data, indent=2, ensure_ascii=False, default=str)) return # --example-dir: entweder direkter Verzeichnispfad oder Unterordnername in PV_EXAMPLES @@ -189,8 +443,17 @@ def main() -> None: log.info("Beispiel-Verzeichnis (einzeln): %s", target_dir) result: dict[str, list[dict[str, Any]]] = {} - _process_single_folder(target_dir, target_dir.parent, result, log) + _process_single_folder( + target_dir, + target_dir.parent, + result, + log, + write_json=args.to_json, + write_csv=args.to_csv, + write_xlsx=args.to_xlsx, + ) log.info("Objekt erzeugt mit %d Schluesseln", len(result)) + #print(json.dumps(result, indent=2, ensure_ascii=False, default=str)) if __name__ == "__main__": diff --git a/lib/extract_products.py b/lib/extract_products.py index 0b230bf..addc175 100644 --- a/lib/extract_products.py +++ b/lib/extract_products.py @@ -81,19 +81,17 @@ def _extract_toc_numbers(doc) -> dict[str, str]: """Extrahiert Kapitelnummern aus dem Inhaltsverzeichnis des PDFs. Sucht TOC-Seiten (Seiten mit '....' Punkt-Fuellern) und ordnet - Ueberschriftentexte ihren Kapitelnummern zu. + Ueberschriftentexte ihren Kapitelnummern zu. Nur Eintraege mit + expliziter Nummer werden erfasst (bis Level 2). Returns: - Dict title_text -> kapitelnummer, z.B. {"Typen von Weichen mit Bögen": "2.3.1"} + Dict title_text -> kapitelnummer, z.B. {"Weichen Omniflo": "2.3"} """ - import fitz - toc_entries: dict[str, str] = {} for page_num in range(len(doc)): page = doc[page_num] text = page.get_text() - # Nur TOC-Seiten verarbeiten (enthalten "....") if "...." not in text: continue @@ -106,8 +104,7 @@ def _extract_toc_numbers(doc) -> dict[str, str]: if not line: continue - # TOC-Zeile: "Text ........... Seitennr" - # Kapitelnummer steht oft auf separater Zeile davor + # Kapitelnummer auf separater Zeile num_match = re.match(r"^(\d+(?:\.\d+)*)\s*$", line) if num_match: current_number = num_match.group(1) @@ -115,32 +112,59 @@ def _extract_toc_numbers(doc) -> dict[str, str]: # Zeile mit Punkt-Fuellern = TOC-Eintrag dot_match = re.search(r"\.{4,}", line) - if dot_match: - # Text vor den Punkten extrahieren - title = line[:dot_match.start()].strip() - # Seitennummer nach den Punkten - if title and current_number: - toc_entries[title] = current_number - current_number = "" - elif title: - # Nummer koennte am Anfang des Titels stehen - nm = re.match(r"^(\d+(?:\.\d+)*)\s+(.+)", title) - if nm: - toc_entries[nm.group(2).strip()] = nm.group(1) + if not dot_match: + continue - # Auch inline Nummern: "1.4 Abkürzungen ..... 8" - for line in lines: - line = line.strip() - dot_match = re.search(r"\.{4,}", line) - if dot_match: - title_part = line[:dot_match.start()].strip() - nm = re.match(r"^(\d+(?:\.\d+)*)\s+(.+)", title_part) - if nm: - toc_entries[nm.group(2).strip()] = nm.group(1) + title = line[:dot_match.start()].strip() + if not title: + continue + + # Inline-Nummer am Anfang? "1.4 Abkürzungen ....." + nm = re.match(r"^(\d+(?:\.\d+)*)\s+(.+)", title) + if nm: + toc_entries[nm.group(2).strip()] = nm.group(1) + current_number = "" + elif current_number: + toc_entries[title] = current_number + current_number = "" return toc_entries +RE_DATE = re.compile(r"^\d{2}\.\d{2}\.\d{4}$") +RE_COPYRIGHT = re.compile(r"^Copyright\s+\d{4}", re.IGNORECASE) +RE_VERSION_LINE = re.compile(r"^Version\s*[:.]?\s*\d", re.IGNORECASE) + +# Bekannte Rausch-Muster: Autorinnen, Metadaten, Copyright-Zeilen +_NOISE_KEYWORDS = { + "annette schopper", "erstellt:", "freigabe:", "nicht freigegeben", + "nur zur information", "schönenberger systeme gmbh", + "justus-von-liebig-str", "info@schoenenberger.de", + "www.schoenenberger.de", +} + + +def _is_noise(text: str) -> bool: + """Erkennt Rausch-Zeilen die nicht ins RST gehoeren.""" + lower = text.lower().strip() + if RE_COPYRIGHT.match(text): + return True + if RE_DATE.match(text.strip()): + return True + if RE_VERSION_LINE.match(text): + return True + for kw in _NOISE_KEYWORDS: + if kw in lower: + return True + # Seitenzahlen + if re.match(r"^Seite\s+\d+$", text, re.IGNORECASE): + return True + # Einzelne kurze Jahreszahlen mit Autorin: "2020 Annette Schopper" + if re.match(r"^\d{4}\s+[A-Z][a-z]+\s+[A-Z][a-z]+$", text): + return True + return False + + def extract_text_to_rst(pdf_path: Path, rst_path: Path) -> int: """Extrahiert Text aus PDF und schreibt ihn als .rst. @@ -188,6 +212,8 @@ def extract_text_to_rst(pdf_path: Path, rst_path: Path) -> int: text = text.strip() if not text: continue + if _is_noise(text): + continue raw_lines.append({ "text": text, @@ -311,6 +337,10 @@ def extract_text_to_rst(pdf_path: Path, rst_path: Path) -> int: # Mehrfache Leerzeilen reduzieren rst = "\n".join(rst_lines) rst = re.sub(r"\n{3,}", "\n\n", rst) + # 9-stellige Nummern mit Leerzeichen zusammenfuegen: "834 372 007" -> "834372007" + rst = re.sub(r"\b(\d{3})\s(\d{3})\s(\d{3})\b", r"\1\2\3", rst) + rst = re.sub(r"\b(\d{3})\s(\d{6})\b", r"\1\2", rst) + rst = re.sub(r"\b(\d{6})\s(\d{3})\b", r"\1\2", rst) rst = rst.strip() + "\n" rst_path.parent.mkdir(parents=True, exist_ok=True)