246 lines
8.3 KiB
Python
246 lines
8.3 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
Einmalige Migration: extrahiert die 403 Eintraege der Texttabelle *ssg-texts*
|
|
aus dem Quelltext von Lisp/ssg_lang.lsp und schreibt sie in
|
|
lang/de_DE.json und lang/en_GB.json.
|
|
|
|
Danach wird ein unabhaengiger Round-Trip-Test gefahren: die geschriebenen
|
|
JSON-Dateien werden mit einer Python-Nachbildung des LISP-JSON-Parsers
|
|
(ssg-cfg-parse-nested + ssg-cfg-parse-value) und der Unescape-Logik
|
|
(ssg-lang-unescape: \\n -> Zeilenumbruch) neu eingelesen und exakt gegen die
|
|
LISP-Laufzeitwerte verglichen.
|
|
|
|
Aufruf (aus dem Repo-Wurzelverzeichnis):
|
|
py tools/migrate_lang_to_json.py
|
|
"""
|
|
import re
|
|
import sys
|
|
import io
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parent.parent
|
|
SRC = REPO / "Lisp" / "ssg_lang.lsp"
|
|
LANG_DIR = REPO / "lang"
|
|
OUT_DE = LANG_DIR / "de_DE.json"
|
|
OUT_EN = LANG_DIR / "en_GB.json"
|
|
|
|
# (list "key" (cons "de_DE" "...") (cons "en_GB" "..."))
|
|
# Wert erlaubt Escape-Sequenzen (\\.) und beliebige Nicht-Quote-Zeichen.
|
|
ENTRY_RE = re.compile(
|
|
r'\(list\s+"([^"]+)"\s*'
|
|
r'\(cons\s+"de_DE"\s+"((?:[^"\\]|\\.)*)"\s*\)\s*'
|
|
r'\(cons\s+"en_GB"\s+"((?:[^"\\]|\\.)*)"\s*\)',
|
|
re.DOTALL,
|
|
)
|
|
|
|
|
|
def extract_entries(text):
|
|
return [(m.group(1), m.group(2), m.group(3)) for m in ENTRY_RE.finditer(text)]
|
|
|
|
|
|
# Erlaubte Backslash-Sequenzen im Quelltext-Wert: \n und \uXXXX.
|
|
ESC_TOKEN_RE = re.compile(r'\\(u[0-9a-fA-F]{4}|.)')
|
|
|
|
|
|
def assert_only_known_escapes(entries):
|
|
"""Erlaubt sind nur \\n und \\uXXXX; jede andere \\X-Sequenz sowie nackte
|
|
Anfuehrungszeichen wuerden die JSON-Erzeugung/-Gueltigkeit gefaehrden.
|
|
Echte Zeilenumbrueche im Wert sind zulaessig (werden zu \\n gewandelt)."""
|
|
bad = []
|
|
for key, de, en in entries:
|
|
for lang, val in (("de_DE", de), ("en_GB", en)):
|
|
for m in ESC_TOKEN_RE.finditer(val):
|
|
tok = m.group(1)
|
|
if not (tok == 'n' or (tok.startswith('u') and len(tok) == 5)):
|
|
bad.append((key, lang, m.group(0)))
|
|
if '"' in val:
|
|
bad.append((key, lang, 'nacktes Anfuehrungszeichen'))
|
|
if bad:
|
|
for b in bad[:20]:
|
|
print(" UNERWARTET:", b, file=sys.stderr)
|
|
raise SystemExit(f"Abbruch: {len(bad)} unerwartete Escapes/Zeichen gefunden.")
|
|
|
|
|
|
def convert_for_json(raw):
|
|
"""Rohwert (LISP-Quelltext) in einen einzeiligen, gueltigen JSON-String-Inhalt
|
|
ueberfuehren:
|
|
- echter Zeilenumbruch -> \\n-Token (Eintrag bleibt einzeilig)
|
|
- \\uXXXX -> echtes Unicode-Zeichen (konsistent zu de_DE '△')
|
|
- vorhandene \\n-Token bleiben unveraendert
|
|
Kein json.dumps, damit die \\n-Token nicht zu \\\\n verdoppelt werden."""
|
|
s = raw.replace('\r\n', '\n').replace('\n', '\\n')
|
|
s = ESC_TOKEN_RE.sub(
|
|
lambda m: chr(int(m.group(1)[1:], 16))
|
|
if (m.group(1).startswith('u') and len(m.group(1)) == 5)
|
|
else m.group(0),
|
|
s)
|
|
return s
|
|
|
|
|
|
def write_json(path, entries, idx):
|
|
"""Schreibt {"texts": {"key": "wert", ...}}."""
|
|
lines = ['{', ' "texts": {']
|
|
n = len(entries)
|
|
for i, (key, de, en) in enumerate(entries):
|
|
val = convert_for_json(de if idx == 1 else en)
|
|
comma = ',' if i < n - 1 else ''
|
|
lines.append(f' "{key}": "{val}"{comma}')
|
|
lines.append(' }')
|
|
lines.append('}')
|
|
with io.open(path, "w", encoding="utf-8", newline="\n") as f:
|
|
f.write("\n".join(lines) + "\n")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Python-Nachbildung des LISP-Parsers fuer den Round-Trip-Test
|
|
# ---------------------------------------------------------------------------
|
|
def lisp_cfg_trim(s):
|
|
s = s.strip(" \t\r\n")
|
|
if s.endswith(","):
|
|
s = s[:-1]
|
|
return s.strip(" \t\r\n")
|
|
|
|
|
|
def lisp_parse_value(val):
|
|
val = lisp_cfg_trim(val)
|
|
if val == "null":
|
|
return None
|
|
if val == "true":
|
|
return True
|
|
if val == "false":
|
|
return None
|
|
if val.startswith('"'):
|
|
if len(val) > 1 and val.endswith('"'):
|
|
return val[1:len(val) - 1]
|
|
return val
|
|
# Zahlen/Arrays kommen in den Sprachdateien nicht vor
|
|
return val
|
|
|
|
|
|
def lisp_parse_kv(zeile):
|
|
zeile = lisp_cfg_trim(zeile)
|
|
if not (len(zeile) > 0 and zeile.startswith('"')):
|
|
return None
|
|
pos = zeile.find('"', 1) # schliessendes Quote des Keys (0-basiert)
|
|
if pos == -1:
|
|
return None
|
|
key = zeile[1:pos]
|
|
rest = zeile[pos + 1:]
|
|
pos2 = rest.find(":")
|
|
if pos2 == -1:
|
|
return None
|
|
val = rest[pos2 + 1:]
|
|
return (key, lisp_parse_value(val))
|
|
|
|
|
|
def lisp_parse_nested(lines):
|
|
"""Nachbildung von ssg-cfg-parse-nested (1 Ebene tief)."""
|
|
ergebnis = {}
|
|
section = None
|
|
in_inner = False
|
|
for zeile in lines:
|
|
trimmed = lisp_cfg_trim(zeile)
|
|
if not in_inner and trimmed == "{":
|
|
continue
|
|
if not in_inner and trimmed == "}":
|
|
continue
|
|
if (not in_inner) and (": {" in trimmed):
|
|
# Section-Start "name": {
|
|
q = trimmed.find('"', 1)
|
|
section = trimmed[1:q]
|
|
ergebnis[section] = {}
|
|
in_inner = True
|
|
continue
|
|
if in_inner and trimmed in ("}", "},"):
|
|
in_inner = False
|
|
section = None
|
|
continue
|
|
if in_inner:
|
|
kv = lisp_parse_kv(trimmed)
|
|
if kv:
|
|
ergebnis[section][kv[0]] = kv[1]
|
|
return ergebnis
|
|
|
|
|
|
def lisp_lang_unescape(val):
|
|
"""Nachbildung von ssg-lang-unescape: \\n -> Zeilenumbruch."""
|
|
if val is None:
|
|
return None
|
|
return val.replace("\\n", "\n")
|
|
|
|
|
|
def lisp_runtime_value(raw):
|
|
"""Der erwartete Laufzeitwert nach der gewaehlten Semantik:
|
|
- \\n und echter Zeilenumbruch -> Zeilenumbruch
|
|
- \\uXXXX -> echtes Unicode-Zeichen
|
|
- \\t \\r \\\\ \\\" defensiv aufgeloest"""
|
|
raw = raw.replace('\r\n', '\n')
|
|
out = []
|
|
i = 0
|
|
while i < len(raw):
|
|
if raw[i] == "\\" and i + 1 < len(raw):
|
|
nxt = raw[i + 1]
|
|
if nxt == "u" and i + 6 <= len(raw) and \
|
|
re.match(r'[0-9a-fA-F]{4}', raw[i + 2:i + 6]):
|
|
out.append(chr(int(raw[i + 2:i + 6], 16))); i += 6; continue
|
|
if nxt == "n":
|
|
out.append("\n"); i += 2; continue
|
|
if nxt == "t":
|
|
out.append("\t"); i += 2; continue
|
|
if nxt == "r":
|
|
out.append("\r"); i += 2; continue
|
|
if nxt in ('"', "\\"):
|
|
out.append(nxt); i += 2; continue
|
|
out.append(raw[i]); i += 1
|
|
return "".join(out)
|
|
|
|
|
|
def roundtrip_check(entries):
|
|
de_tree = lisp_parse_nested(
|
|
io.open(OUT_DE, encoding="utf-8").read().splitlines())
|
|
en_tree = lisp_parse_nested(
|
|
io.open(OUT_EN, encoding="utf-8").read().splitlines())
|
|
de_sec = de_tree.get("texts", {})
|
|
en_sec = en_tree.get("texts", {})
|
|
assert len(de_sec) == len(entries), \
|
|
f"de_DE: {len(de_sec)} Keys re-geparst, erwartet {len(entries)}"
|
|
assert len(en_sec) == len(entries), \
|
|
f"en_GB: {len(en_sec)} Keys re-geparst, erwartet {len(entries)}"
|
|
fails = 0
|
|
for key, de_raw, en_raw in entries:
|
|
want_de = lisp_runtime_value(de_raw)
|
|
want_en = lisp_runtime_value(en_raw)
|
|
got_de = lisp_lang_unescape(de_sec.get(key))
|
|
got_en = lisp_lang_unescape(en_sec.get(key))
|
|
if got_de != want_de:
|
|
fails += 1
|
|
print(f" DE-DIFF {key!r}: {want_de!r} != {got_de!r}", file=sys.stderr)
|
|
if got_en != want_en:
|
|
fails += 1
|
|
print(f" EN-DIFF {key!r}: {want_en!r} != {got_en!r}", file=sys.stderr)
|
|
if fails:
|
|
raise SystemExit(f"Round-Trip fehlgeschlagen: {fails} Abweichungen.")
|
|
print("Round-Trip OK: alle Werte identisch nach Re-Parse.")
|
|
|
|
|
|
def main():
|
|
text = io.open(SRC, encoding="utf-8").read()
|
|
entries = extract_entries(text)
|
|
print(f"{len(entries)} Eintraege extrahiert.")
|
|
assert len(entries) == 403, f"Erwartet 403, gefunden {len(entries)}"
|
|
keys = [k for k, _, _ in entries]
|
|
assert len(set(keys)) == len(keys), "Duplikat-Keys gefunden"
|
|
assert_only_known_escapes(entries)
|
|
|
|
LANG_DIR.mkdir(exist_ok=True)
|
|
write_json(OUT_DE, entries, 1)
|
|
write_json(OUT_EN, entries, 2)
|
|
print(f"Geschrieben: {OUT_DE.relative_to(REPO)}, {OUT_EN.relative_to(REPO)}")
|
|
|
|
roundtrip_check(entries)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|