diff --git a/.gitignore b/.gitignore index 5186ceae..70a6d1c6 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ vcpkg.json # Local Python (setup.bat / CMake) for embedding tools/python314/ +__pycache__/ +*.pyc diff --git a/tools/editor/drive_headers.py b/tools/editor/drive_headers.py index 85550274..16af94fb 100644 --- a/tools/editor/drive_headers.py +++ b/tools/editor/drive_headers.py @@ -8,11 +8,9 @@ without --write it only reports what it would write; --zip packs the written tem import os import re import sys -import tarfile -import time -import zipfile -LINEMARK = '//$e' +from editor_headers import body_lines, decode, load, options, pack, write + FOLDER = 'scenery/tory/napedy' PATTERN = re.compile(r'^(ele1?|mech[12]?)-rz([lp])-([lp])((?:_[a-z]+)*)\.inc$') KINDS = {'ele': 'elektryczny', 'mech': 'mechaniczny'} @@ -24,15 +22,9 @@ def examine(path, file): m = PATTERN.match(file) if m is None: return None, 'not a drive of a single switch' - raw = open(path, 'rb').read() # NOSONAR - try: - text = raw.decode('utf-8') - encoding = 'utf-8' if raw.startswith(b'\xef\xbb\xbf') or max(raw, default=0) > 127 else 'cp1250' - except UnicodeDecodeError: - text = raw.decode('cp1250') - encoding = 'cp1250' - lines = [line for line in text.splitlines() if not line.startswith(LINEMARK)] - body = '\n'.join(lines).lower() + raw = load(path) + text, encoding = decode(raw) + body = '\n'.join(body_lines(text)).lower() count = max([int(n) for n in re.findall(r'\(p(\d+)\)', body)] or [0]) if count != 7 or 'origin (p2) (p3) (p4)' not in body or 'rotate (p6) (p5) (p7)' not in body: return None, 'not origin (p2) (p3) (p4) rotate (p6) (p5) (p7)' @@ -63,26 +55,12 @@ def examine(path, file): return {'raw': raw, 'text': text, 'encoding': encoding, 'header': header, 'family': family, 'hand': hand, 'side': side}, '' -def rewrite(info): - text = info['text'] - eol = '\r\n' if '\r\n' in text else '\n' - bom = text.startswith('') - if bom: - text = text[1:] - kept = [line for line in text.split(eol) if not line.startswith(LINEMARK)] - block = [LINEMARK + ' ' + line for line in info['header']] - out = eol.join(block + kept) - return (('' if bom else '') + out).encode(info['encoding']) - - def main(): - if len(sys.argv) < 2: - print(__doc__) + chosen = options(sys.argv, __doc__) + if chosen is None: return 1 - data = sys.argv[1] + data, writing, target = chosen folder = os.path.join(data, FOLDER) - write = '--write' in sys.argv - pack = sys.argv[sys.argv.index('--zip') + 1] if '--zip' in sys.argv else None done = [] skipped = [] for file in sorted(os.listdir(folder)): @@ -98,22 +76,10 @@ def main(): for file, reason in skipped: print('skipped %s: %s' % (file, reason)) print('%d described, %d skipped' % (len(done), len(skipped))) - if write: - backup = os.path.join(data, 'editor_backup') - os.makedirs(backup, exist_ok=True) - archive = os.path.join(backup, 'drives_%s.tar' % time.strftime('%Y%m%d_%H%M%S')) - with tarfile.open(archive, 'w') as tar: - for file, _ in done: - tar.add(os.path.join(folder, file), arcname=FOLDER + '/' + file) - for file, info in done: - with open(os.path.join(folder, file), 'wb') as out: - out.write(rewrite(info)) - print('written, originals in ' + archive) - if pack: - with zipfile.ZipFile(pack, 'w', zipfile.ZIP_DEFLATED) as out: - for file, info in done: - out.writestr(FOLDER + '/' + file, rewrite(info)) - print('packed into ' + pack) + if writing: + write(data, FOLDER, 'drives', done) + if target: + pack(target, FOLDER, done) return 0 diff --git a/tools/editor/editor_headers.py b/tools/editor/editor_headers.py new file mode 100644 index 00000000..54b6e720 --- /dev/null +++ b/tools/editor/editor_headers.py @@ -0,0 +1,74 @@ +"""Shared parts of the scripts writing //$e descriptions into the templates of the scenery.""" + +import os +import tarfile +import time +import zipfile + +LINEMARK = '//$e' +BOM = '' + + +def load(path): + with open(path, 'rb') as source: # NOSONAR + return source.read() + + +def decode(raw, plain='cp1250'): + try: + text = raw.decode('utf-8') + encoding = 'utf-8' if raw.startswith(b'\xef\xbb\xbf') or max(raw, default=0) > 127 else plain + except UnicodeDecodeError: + text = raw.decode('cp1250') + encoding = 'cp1250' + return text, encoding + + +def body_lines(text): + return [line for line in text.splitlines() if not line.startswith(LINEMARK)] + + +def rewrite(info): + if info.get('header') is None: + return info['raw'] + text = info['text'] + eol = '\r\n' if '\r\n' in text else '\n' + bom = text.startswith(BOM) + if bom: + text = text[1:] + kept = [line for line in text.split(eol) if not line.startswith(LINEMARK)] + block = [LINEMARK + ' ' + line for line in info['header']] + out = eol.join(block + kept) + return ((BOM if bom else '') + out).encode(info['encoding']) + + +def options(argv, usage): + if len(argv) < 2: + print(usage) + return None + pack = argv[argv.index('--zip') + 1] if '--zip' in argv else None + return argv[1], '--write' in argv, pack + + +def write(data, folder, name, entries): + backup = os.path.join(data, 'editor_backup') + os.makedirs(backup, exist_ok=True) # NOSONAR + archive = os.path.join(backup, time.strftime(name + '_%Y%m%d_%H%M%S.tar')) + root = os.path.join(data, folder) + with tarfile.open(archive, 'w') as tar: # NOSONAR + for file, _ in entries: + tar.add(os.path.join(root, file), arcname=folder + '/' + file) # NOSONAR + for file, info in entries: + content = rewrite(info) + if content != info['raw']: + with open(os.path.join(root, file), 'wb') as out: # NOSONAR + out.write(content) + print('written, originals in', archive) + + +def pack(target, folder, entries): + with zipfile.ZipFile(target, 'w', zipfile.ZIP_DEFLATED) as out: # NOSONAR + for file, info in entries: + out.writestr(folder + '/' + file, rewrite(info)) + print('packed into ' + target) + diff --git a/tools/editor/signal_headers.py b/tools/editor/signal_headers.py index 000195ea..63aacf75 100644 --- a/tools/editor/signal_headers.py +++ b/tools/editor/signal_headers.py @@ -8,8 +8,8 @@ without --write it only reports what it would write import os import re import sys -import tarfile -import time + +from editor_headers import body_lines, decode, load, write FAMILIES = [ (re.compile(r'^ss[dow]?\d'), 'main'), @@ -24,8 +24,6 @@ FAMILIES = [ COLOURS = [('zielon', 'z'), ('czerwon', 'c'), ('pomara', 'p'), ('żółt', 'p'), ('biał', 'b'), ('mleczn', 'b'), ('niebiesk', 'n'), ('ślep', 'x'), ('brak', 'x')] -LINEMARK = '//$e' - def tokens(text): text = re.sub(r'//[^\n]*', ' ', text) @@ -50,14 +48,9 @@ def statements(words, keyword): def examine(path, file): - raw = open(path, 'rb').read() # NOSONAR - try: - text = raw.decode('utf-8') - encoding = 'utf-8' - except UnicodeDecodeError: - text = raw.decode('cp1250') - encoding = 'cp1250' - lines = [line for line in text.splitlines() if not line.startswith(LINEMARK)] + raw = load(path) + text, encoding = decode(raw, 'utf-8') + lines = body_lines(text) body = '\n'.join(lines) words = tokens(body) count = max([int(m) for m in re.findall(r'\(p(\d+)\)', body)] or [0]) @@ -176,24 +169,12 @@ def examine(path, file): return {'raw': raw, 'text': text, 'encoding': encoding, 'header': header, 'kind': kind, 'model': model, 'read': read, 'lamps': lamps, 'mount': mount, 'lean': lean, 'count': count}, '' -def rewrite(info): - text = info['text'] - eol = '\r\n' if '\r\n' in text else '\n' - bom = text.startswith('') - if bom: - text = text[1:] - kept = [line for line in text.split(eol) if not line.startswith(LINEMARK)] - block = [LINEMARK + ' ' + line for line in info['header']] - out = eol.join(block + kept) - return (('' if bom else '') + out).encode(info['encoding']) - - def main(): if len(sys.argv) < 2: print(__doc__) return 1 root = os.path.join(sys.argv[1], 'scenery') - write = '--write' in sys.argv + writing = '--write' in sys.argv files = sorted(f for f in os.listdir(root) if f.endswith('.inc')) files += ['pkp/semafory/' + f for f in sorted(os.listdir(os.path.join(root, 'pkp', 'semafory'))) if f.endswith('.inc')] done = [] @@ -217,19 +198,8 @@ def main(): kinds[info['kind']] = kinds.get(info['kind'], 0) + 1 print('described: %d %s, skipped: %d' % (len(done), kinds, len(skipped))) - if write: - backup = os.path.join(sys.argv[1], 'editor_backup') - os.makedirs(backup, exist_ok=True) # NOSONAR - archive = os.path.join(backup, time.strftime('signals_%Y%m%d_%H%M%S.tar')) - with tarfile.open(archive, 'w') as tar: # NOSONAR - for file, _ in done: - tar.add(os.path.join(root, file), arcname='scenery/' + file) # NOSONAR - for file, info in done: - content = rewrite(info) - if content != info['raw']: - with open(os.path.join(root, file), 'wb') as out: # NOSONAR - out.write(content) - print('written, originals in', archive) + if writing: + write(sys.argv[1], 'scenery', 'signals', done) return 0 diff --git a/tools/editor/turntable_headers.py b/tools/editor/turntable_headers.py index 8a81b128..891ad432 100755 --- a/tools/editor/turntable_headers.py +++ b/tools/editor/turntable_headers.py @@ -8,35 +8,23 @@ without --write it only reports what it would write; --zip packs every described import os import re import sys -import tarfile -import time -import zipfile -LINEMARK = '//$e' +from editor_headers import LINEMARK, body_lines, decode, load, options, pack, write + FOLDER = 'scenery' ROTATION = re.compile(r'^event\s+\S*tor(\d+)rot\s+animation\s+\S+\s+\S+\s+rotate\s+\S+\s+\S+\s+\S+\s+(\S+)', re.I | re.M) MODEL = re.compile(r'model\s+0\s+0\s+0\s+0\s+\S*?obrotnica(\d+)m\.t3d\s+\(p6\)', re.I) -def decode(raw): - try: - text = raw.decode('utf-8') - encoding = 'utf-8' if raw.startswith(b'\xef\xbb\xbf') or max(raw, default=0) > 127 else 'cp1250' - except UnicodeDecodeError: - text = raw.decode('cp1250') - encoding = 'cp1250' - return text, encoding - - def examine(path, relative): - raw = open(path, 'rb').read() # NOSONAR + raw = load(path) if re.search(rb'tor\d+rot\s+animation', raw, re.I) is None: return None, None text, encoding = decode(raw) marks = [line for line in text.splitlines() if line.startswith(LINEMARK)] if any('category: turntable' in line for line in marks): return {'raw': raw, 'header': None, 'described': True}, '' - body = '\n'.join(line for line in text.splitlines() if not line.startswith(LINEMARK)) + body = '\n'.join(body_lines(text)) lower = body.lower() if 'origin (p2) (p3) (p4)' not in lower or 'rotate 0 (p5) 0' not in lower: return None, 'not origin (p2) (p3) (p4) rotate 0 (p5) 0' @@ -75,28 +63,12 @@ def examine(path, relative): return {'raw': raw, 'text': text, 'encoding': encoding, 'header': header, 'described': False, 'length': length, 'angles': angles}, '' -def rewrite(info): - if info['header'] is None: - return info['raw'] - text = info['text'] - eol = '\r\n' if '\r\n' in text else '\n' - bom = text.startswith('') - if bom: - text = text[1:] - kept = [line for line in text.split(eol) if not line.startswith(LINEMARK)] - block = [LINEMARK + ' ' + line for line in info['header']] - out = eol.join(block + kept) - return (('' if bom else '') + out).encode(info['encoding']) - - def main(): - if len(sys.argv) < 2: - print(__doc__) + chosen = options(sys.argv, __doc__) + if chosen is None: return 1 - data = sys.argv[1] + data, writing, target = chosen folder = os.path.join(data, FOLDER) - write = '--write' in sys.argv - pack = sys.argv[sys.argv.index('--zip') + 1] if '--zip' in sys.argv else None done = [] skipped = [] for root, _, files in os.walk(folder): @@ -119,22 +91,10 @@ def main(): print('skipped %s: %s' % (file, reason)) fresh = [(file, info) for file, info in done if not info['described']] print('%d described now, %d already described, %d skipped' % (len(fresh), len(done) - len(fresh), len(skipped))) - if write and fresh: - backup = os.path.join(data, 'editor_backup') - os.makedirs(backup, exist_ok=True) - archive = os.path.join(backup, 'turntables_%s.tar' % time.strftime('%Y%m%d_%H%M%S')) - with tarfile.open(archive, 'w') as tar: - for file, _ in fresh: - tar.add(os.path.join(folder, file), arcname=FOLDER + '/' + file) - for file, info in fresh: - with open(os.path.join(folder, file), 'wb') as out: - out.write(rewrite(info)) - print('written, originals in ' + archive) - if pack: - with zipfile.ZipFile(pack, 'w', zipfile.ZIP_DEFLATED) as out: - for file, info in done: - out.writestr(FOLDER + '/' + file, rewrite(info)) - print('packed into ' + pack) + if writing and fresh: + write(data, FOLDER, 'turntables', fresh) + if target: + pack(target, FOLDER, done) return 0