483 lines
19 KiB
Python
483 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Test runner for kabellaengen project
|
|
Executes getexdraw.bat for all DXF files in testdata directory and compares outputs
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import subprocess
|
|
import glob
|
|
from pathlib import Path
|
|
import difflib
|
|
import argparse
|
|
|
|
|
|
class TestResult:
|
|
def __init__(self, test_name):
|
|
self.test_name = test_name
|
|
self.passed = False
|
|
self.error_msg = ""
|
|
self.differences = []
|
|
|
|
|
|
class TestRunner:
|
|
def __init__(self, create_references=False, mode='getexdraw', clean=False, target_file=None):
|
|
# Use environment variables set by setenv.bat
|
|
self.project_root = Path(os.environ['PROJECT'])
|
|
self.testdata_dir = Path(os.environ.get('PROJECT_TEST', self.project_root / "testdata"))
|
|
self.work_dir = Path(os.environ.get('PROJECT_WORK', self.project_root / "work"))
|
|
self.bin_dir = Path(os.environ.get('PROJECT_BIN', self.project_root / "bin"))
|
|
self.lib_dir = Path(os.environ.get('PROJECT_LIB', self.project_root / "lib"))
|
|
self.cfg_dir = Path(os.environ.get('PROJECT_CFG', self.project_root / "cfg"))
|
|
self.getexdraw_bat = self.bin_dir / "getexdraw.bat"
|
|
self.ioconverter_bat = self.bin_dir / "ioconverter.bat"
|
|
self.translate_py = self.lib_dir / "translate.py"
|
|
|
|
self.test_results = []
|
|
self.failed_tests = []
|
|
self.create_references = create_references
|
|
self.mode = mode
|
|
self.clean = clean
|
|
self.target_file = target_file
|
|
|
|
def find_test_files(self):
|
|
"""Find DXF files in testdata directory - all files or specific file"""
|
|
if self.target_file:
|
|
# Remove .dxf suffix if present to avoid double extension
|
|
clean_target = self.target_file
|
|
if clean_target.endswith('.dxf'):
|
|
clean_target = clean_target[:-4]
|
|
|
|
# Check if specific file exists
|
|
target_dxf = self.testdata_dir / f"{clean_target}.dxf"
|
|
if target_dxf.exists():
|
|
return [clean_target]
|
|
else:
|
|
print(f"Error: Test file {target_dxf} not found")
|
|
return []
|
|
else:
|
|
# Return all DXF files
|
|
dxf_files = list(self.testdata_dir.glob("*.dxf"))
|
|
return [f.stem for f in dxf_files] # Return filenames without extension
|
|
|
|
def run_getexdraw(self, test_filename):
|
|
"""Execute getexdraw.bat for given test file"""
|
|
dxf_file = self.testdata_dir / f"{test_filename}.dxf"
|
|
if not dxf_file.exists():
|
|
return False, f"Test file {dxf_file} not found"
|
|
|
|
try:
|
|
# Change to bin directory for execution (where batch scripts expect to run from)
|
|
os.chdir(self.bin_dir)
|
|
|
|
# Execute getexdraw.bat with automated input to handle pause
|
|
cmd = [str(self.getexdraw_bat), str(dxf_file)]
|
|
# Send newline to handle the pause command
|
|
result = subprocess.run(cmd, capture_output=True, text=True, shell=True, input="\n")
|
|
|
|
# Check if the expected output files were created instead of relying on return code
|
|
# The batch script always exits with code 1 due to interactive design, but files are created
|
|
expected_positions_file = self.work_dir / test_filename / f"{test_filename}_positionsdraw.json"
|
|
expected_todraw_file = self.work_dir / test_filename / f"{test_filename}_todraw.json"
|
|
expected_errors_file = self.work_dir / test_filename / f"{test_filename}_errors.json"
|
|
|
|
# Success case: both normal files exist
|
|
if expected_positions_file.exists() and expected_todraw_file.exists():
|
|
return True, "Success"
|
|
# Error case: positions file missing but errors file exists
|
|
elif not expected_positions_file.exists() and expected_errors_file.exists():
|
|
return True, "Success (with errors)"
|
|
else:
|
|
missing_files = []
|
|
if not expected_positions_file.exists() and not expected_errors_file.exists():
|
|
missing_files.append(f"{test_filename}_positionsdraw.json or {test_filename}_errors.json")
|
|
if not expected_todraw_file.exists():
|
|
missing_files.append(str(expected_todraw_file))
|
|
return False, f"getexdraw.bat failed - expected output files not found: {', '.join(missing_files)}\nSTDOUT: {result.stdout}\nSTDERR: {result.stderr}"
|
|
|
|
except Exception as e:
|
|
return False, f"Exception during execution: {str(e)}"
|
|
|
|
def run_ioconverter(self, test_filename):
|
|
"""Execute ioconverter.bat for given test file"""
|
|
dxf_file = self.testdata_dir / f"{test_filename}.dxf"
|
|
if not dxf_file.exists():
|
|
return False, f"Test file {dxf_file} not found"
|
|
|
|
try:
|
|
# Change to bin directory for execution (where batch scripts expect to run from)
|
|
os.chdir(self.bin_dir)
|
|
|
|
# Execute ioconverter.bat with automated input to handle pause
|
|
cmd = [str(self.ioconverter_bat), str(dxf_file)]
|
|
# Send newline to handle the pause command
|
|
result = subprocess.run(cmd, capture_output=True, text=True, shell=True, input="\n")
|
|
|
|
# Check if the expected output files were created
|
|
expected_positions_file = self.work_dir / test_filename / f"{test_filename}_positionsconv.json"
|
|
expected_errors_file = self.work_dir / test_filename / f"{test_filename}_errors.json"
|
|
|
|
# Success case: positions file exists
|
|
if expected_positions_file.exists():
|
|
return True, "Success"
|
|
# Error case: positions file missing but errors file exists
|
|
elif expected_errors_file.exists():
|
|
return True, "Success (with errors)"
|
|
else:
|
|
return False, f"ioconverter.bat failed - expected output file not found: {expected_positions_file} or {expected_errors_file}\nSTDOUT: {result.stdout}\nSTDERR: {result.stderr}"
|
|
|
|
except Exception as e:
|
|
return False, f"Exception during execution: {str(e)}"
|
|
|
|
def run_translate(self, test_filename):
|
|
"""Execute translate.py for given test file with three variants: raw, cs, multilang"""
|
|
dxf_file = self.testdata_dir / f"{test_filename}.dxf"
|
|
if not dxf_file.exists():
|
|
return False, f"Test file {dxf_file} not found"
|
|
|
|
try:
|
|
# Set up environment for translate.py
|
|
env = os.environ.copy()
|
|
env['PROJECT_TRANSLATION'] = str(self.cfg_dir / "translation")
|
|
|
|
# Variant 1: Raw extraction (no translation)
|
|
cmd_raw = [
|
|
sys.executable,
|
|
str(self.translate_py),
|
|
'--extract',
|
|
'--filename', str(dxf_file),
|
|
'--export-type', 'json',
|
|
'--outname', f'{test_filename}_raw'
|
|
]
|
|
result_raw = subprocess.run(cmd_raw, capture_output=True, text=True, env=env)
|
|
|
|
# Variant 2: CS translation (single language)
|
|
cmd_cs = [
|
|
sys.executable,
|
|
str(self.translate_py),
|
|
'--extract',
|
|
'--filename', str(dxf_file),
|
|
'--export-type', 'json',
|
|
'--outname', f'{test_filename}_cs',
|
|
'--translate', 'CS'
|
|
]
|
|
result_cs = subprocess.run(cmd_cs, capture_output=True, text=True, env=env)
|
|
|
|
# Variant 3: Multilang translation (CS,EN)
|
|
cmd_multilang = [
|
|
sys.executable,
|
|
str(self.translate_py),
|
|
'--extract',
|
|
'--filename', str(dxf_file),
|
|
'--export-type', 'json',
|
|
'--outname', f'{test_filename}_multilang',
|
|
'--translate', 'CS,EN'
|
|
]
|
|
result_multilang = subprocess.run(cmd_multilang, capture_output=True, text=True, env=env)
|
|
|
|
# Check if all expected output files were created
|
|
expected_raw = self.work_dir / f"{test_filename}_raw.json"
|
|
expected_cs = self.work_dir / f"{test_filename}_cs.json"
|
|
expected_multilang = self.work_dir / f"{test_filename}_multilang.json"
|
|
|
|
missing_files = []
|
|
if not expected_raw.exists():
|
|
missing_files.append(f"{test_filename}_raw.json")
|
|
if not expected_cs.exists():
|
|
missing_files.append(f"{test_filename}_cs.json")
|
|
if not expected_multilang.exists():
|
|
missing_files.append(f"{test_filename}_multilang.json")
|
|
|
|
if missing_files:
|
|
error_msg = f"translate.py failed - expected output files not found: {', '.join(missing_files)}"
|
|
if result_raw.returncode != 0:
|
|
error_msg += f"\n\nRaw variant STDOUT:\n{result_raw.stdout}\nSTDERR:\n{result_raw.stderr}"
|
|
if result_cs.returncode != 0:
|
|
error_msg += f"\n\nCS variant STDOUT:\n{result_cs.stdout}\nSTDERR:\n{result_cs.stderr}"
|
|
if result_multilang.returncode != 0:
|
|
error_msg += f"\n\nMultilang variant STDOUT:\n{result_multilang.stdout}\nSTDERR:\n{result_multilang.stderr}"
|
|
return False, error_msg
|
|
else:
|
|
return True, "Success"
|
|
|
|
except Exception as e:
|
|
return False, f"Exception during execution: {str(e)}"
|
|
|
|
def cleanup_test_files(self, test_filename):
|
|
"""Remove the work folder for the given test"""
|
|
if not self.clean:
|
|
return
|
|
|
|
test_work_dir = self.work_dir / test_filename
|
|
if test_work_dir.exists():
|
|
try:
|
|
import shutil
|
|
shutil.rmtree(test_work_dir)
|
|
print(f" [cleaned up {test_work_dir}]")
|
|
except Exception as e:
|
|
print(f" [cleanup failed: {e}]")
|
|
|
|
def compare_json_files(self, test_filename):
|
|
"""Compare generated JSON files with reference JSON files"""
|
|
all_diffs = []
|
|
|
|
# Check what files actually exist and determine comparison strategy based on mode
|
|
errors_file = self.work_dir / test_filename / f"{test_filename}_errors.json"
|
|
|
|
if self.mode == 'translation':
|
|
# For translation mode: compare _raw, _cs, and _multilang variants
|
|
file_suffixes = ['_raw', '_cs', '_multilang']
|
|
# Translation files are directly in work_dir, not in subdirectory
|
|
elif self.mode == 'ioconverter':
|
|
# For ioconverter mode: check for _positionsconv.json
|
|
positions_file = self.work_dir / test_filename / f"{test_filename}_positionsconv.json"
|
|
|
|
# Determine which files to compare based on what exists
|
|
if positions_file.exists():
|
|
# Normal case: compare _positionsconv only
|
|
file_suffixes = ['_positionsconv']
|
|
elif errors_file.exists():
|
|
# Error case: compare _errors only
|
|
file_suffixes = ['_errors']
|
|
else:
|
|
return False, f"No generated files found for {test_filename}", []
|
|
else:
|
|
# For getexdraw mode: check for _positionsdraw.json and _todraw.json
|
|
positions_file = self.work_dir / test_filename / f"{test_filename}_positionsdraw.json"
|
|
todraw_file = self.work_dir / test_filename / f"{test_filename}_todraw.json"
|
|
|
|
# Determine which files to compare based on what exists
|
|
if positions_file.exists():
|
|
# Normal case: compare _positionsdraw and _todraw
|
|
file_suffixes = ['_positionsdraw', '_todraw']
|
|
elif errors_file.exists():
|
|
# Error case: compare _errors only (no _todraw expected when errors occur)
|
|
file_suffixes = ['_errors']
|
|
else:
|
|
return False, f"No generated files found for {test_filename}", []
|
|
|
|
for suffix in file_suffixes:
|
|
# For translation mode, files are directly in work_dir
|
|
if self.mode == 'translation':
|
|
generated_file = self.work_dir / f"{test_filename}{suffix}.json"
|
|
else:
|
|
# For other modes, files are in work/<testfilename>/ subdirectory
|
|
generated_file = self.work_dir / test_filename / f"{test_filename}{suffix}.json"
|
|
|
|
# Reference file should be in testdata/<testfilename><suffix>.json
|
|
reference_file = self.testdata_dir / f"{test_filename}{suffix}.json"
|
|
|
|
if not generated_file.exists():
|
|
return False, f"Generated file {generated_file} not found", []
|
|
|
|
if not reference_file.exists():
|
|
if self.create_references:
|
|
# Copy generated file as reference
|
|
import shutil
|
|
shutil.copy2(generated_file, reference_file)
|
|
print(f"\nCreated reference file: {reference_file}")
|
|
continue
|
|
else:
|
|
return False, f"Reference file {reference_file} not found", []
|
|
|
|
try:
|
|
# Load and compare JSON files
|
|
with open(generated_file, 'r', encoding='utf-8') as f:
|
|
generated_data = json.load(f)
|
|
|
|
with open(reference_file, 'r', encoding='utf-8') as f:
|
|
reference_data = json.load(f)
|
|
|
|
if generated_data != reference_data:
|
|
# Generate diff for debugging
|
|
generated_str = json.dumps(generated_data, indent=2, sort_keys=True)
|
|
reference_str = json.dumps(reference_data, indent=2, sort_keys=True)
|
|
|
|
diff = list(difflib.unified_diff(
|
|
reference_str.splitlines(keepends=True),
|
|
generated_str.splitlines(keepends=True),
|
|
fromfile=str(reference_file),
|
|
tofile=str(generated_file),
|
|
n=3
|
|
))
|
|
|
|
all_diffs.extend([f"\n=== Differences in {suffix}.json ==="])
|
|
all_diffs.extend(diff)
|
|
|
|
except json.JSONDecodeError as e:
|
|
return False, f"JSON decode error in {suffix}.json: {str(e)}", []
|
|
except Exception as e:
|
|
return False, f"Exception during comparison of {suffix}.json: {str(e)}", []
|
|
|
|
if all_diffs:
|
|
return False, "Files differ", all_diffs
|
|
else:
|
|
return True, "All files match", []
|
|
|
|
def run_single_test(self, test_filename):
|
|
"""Run a single test case"""
|
|
print(f"Running test: {test_filename} ... ", end="")
|
|
|
|
result = TestResult(test_filename)
|
|
|
|
# Step 1: Run the appropriate script based on mode
|
|
if self.mode == 'translation':
|
|
success, msg = self.run_translate(test_filename)
|
|
elif self.mode == 'ioconverter':
|
|
success, msg = self.run_ioconverter(test_filename)
|
|
else:
|
|
success, msg = self.run_getexdraw(test_filename)
|
|
|
|
if not success:
|
|
result.error_msg = f"Execution failed: {msg}"
|
|
print("FAIL", end="")
|
|
self.cleanup_test_files(test_filename)
|
|
if not self.clean:
|
|
print()
|
|
return result
|
|
|
|
# Step 2: Compare JSON files
|
|
success, msg, diff = self.compare_json_files(test_filename)
|
|
if not success:
|
|
result.error_msg = f"Comparison failed: {msg}"
|
|
result.differences = diff
|
|
print("FAIL", end="")
|
|
self.cleanup_test_files(test_filename)
|
|
if not self.clean:
|
|
print()
|
|
return result
|
|
|
|
result.passed = True
|
|
print("OK", end="")
|
|
|
|
# Step 3: Cleanup if requested
|
|
self.cleanup_test_files(test_filename)
|
|
|
|
if not self.clean:
|
|
print() # Add newline if no cleanup message was printed
|
|
|
|
return result
|
|
|
|
def run_all_tests(self):
|
|
"""Run all tests and collect results"""
|
|
test_files = self.find_test_files()
|
|
|
|
if not test_files:
|
|
if self.target_file:
|
|
print(f"Test file {self.target_file}.dxf not found in testdata directory")
|
|
else:
|
|
print("No test files found in testdata directory")
|
|
return False
|
|
|
|
if self.target_file:
|
|
print(f"Running test for file: {self.target_file}")
|
|
else:
|
|
print(f"Found {len(test_files)} test files")
|
|
print("=" * 70)
|
|
|
|
for test_file in test_files:
|
|
result = self.run_single_test(test_file)
|
|
self.test_results.append(result)
|
|
|
|
if not result.passed:
|
|
self.failed_tests.append(test_file)
|
|
|
|
print("=" * 70)
|
|
return self.print_summary()
|
|
|
|
def print_summary(self):
|
|
"""Print test summary similar to unittest"""
|
|
total_tests = len(self.test_results)
|
|
passed_tests = sum(1 for r in self.test_results if r.passed)
|
|
failed_tests = total_tests - passed_tests
|
|
|
|
print(f"\nRan {total_tests} tests in {self.testdata_dir}")
|
|
print()
|
|
|
|
if failed_tests > 0:
|
|
print("FAILURES:")
|
|
print("=" * 70)
|
|
|
|
for result in self.test_results:
|
|
if not result.passed:
|
|
print(f"FAIL: {result.test_name}")
|
|
print("-" * 70)
|
|
print(result.error_msg)
|
|
|
|
if result.differences:
|
|
print("\nDifferences:")
|
|
for line in result.differences[:50]: # Limit diff output
|
|
print(line.rstrip())
|
|
if len(result.differences) > 50:
|
|
print(f"... ({len(result.differences) - 50} more lines)")
|
|
print()
|
|
|
|
print("=" * 70)
|
|
print(f"FAILED (failures={failed_tests})")
|
|
|
|
print(f"\nFailed test files: {', '.join(self.failed_tests)}")
|
|
else:
|
|
print("OK")
|
|
|
|
return failed_tests == 0
|
|
|
|
|
|
def main():
|
|
"""Main entry point"""
|
|
parser = argparse.ArgumentParser(
|
|
description="Runs batch scripts for all DXF files in testdata and compares results"
|
|
)
|
|
parser.add_argument(
|
|
'--create-references',
|
|
action='store_true',
|
|
help='Create reference JSON files from current outputs'
|
|
)
|
|
parser.add_argument(
|
|
'--clean',
|
|
action='store_true',
|
|
help='Remove work/filename folders after each test run'
|
|
)
|
|
parser.add_argument(
|
|
'--file',
|
|
type=str,
|
|
help='Run test for specific filename (without .dxf extension)'
|
|
)
|
|
|
|
# Mode selection (mutually exclusive)
|
|
mode_group = parser.add_mutually_exclusive_group()
|
|
mode_group.add_argument(
|
|
'--getexdraw',
|
|
action='store_true',
|
|
help='Run getexdraw.bat tests (default)'
|
|
)
|
|
mode_group.add_argument(
|
|
'--ioconverter',
|
|
action='store_true',
|
|
help='Run ioconverter.bat tests'
|
|
)
|
|
mode_group.add_argument(
|
|
'--translation',
|
|
action='store_true',
|
|
help='Run translate.py tests (extract text with raw, CS, and multilang variants)'
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
# Determine mode - default to getexdraw if none is specified
|
|
if args.translation:
|
|
mode = 'translation'
|
|
elif args.ioconverter:
|
|
mode = 'ioconverter'
|
|
else:
|
|
mode = 'getexdraw'
|
|
|
|
runner = TestRunner(create_references=args.create_references, mode=mode, clean=args.clean, target_file=args.file)
|
|
success = runner.run_all_tests()
|
|
|
|
return 0 if success else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main()) |