refactor(editor): the //$e header scripts share reading, decoding, the rewrite of the header, the backup and the zip in tools/editor/editor_headers.py instead of three copies; reports, zips and written files unchanged

.gitignore

@@ -24,3 +24,5 @@ vcpkg.json
 
 # Local Python (setup.bat / CMake) for embedding
 tools/python314/
+__pycache__/
+*.pyc

tools/editor/drive_headers.py

@@ -8,11 +8,9 @@ without --write it only reports what it would write; --zip packs the written tem
 import os
 import re
 import sys
-import tarfile
-import time
-import zipfile
 
-LINEMARK = '//$e'
+from editor_headers import body_lines, decode, load, options, pack, write
+
 FOLDER = 'scenery/tory/napedy'
 PATTERN = re.compile(r'^(ele1?|mech[12]?)-rz([lp])-([lp])((?:_[a-z]+)*)\.inc$')
 KINDS = {'ele': 'elektryczny', 'mech': 'mechaniczny'}
@@ -24,15 +22,9 @@ def examine(path, file):
     m = PATTERN.match(file)
     if m is None:
         return None, 'not a drive of a single switch'
-    raw = open(path, 'rb').read()  # NOSONAR
-    try:
-        text = raw.decode('utf-8')
-        encoding = 'utf-8' if raw.startswith(b'\xef\xbb\xbf') or max(raw, default=0) > 127 else 'cp1250'
-    except UnicodeDecodeError:
-        text = raw.decode('cp1250')
-        encoding = 'cp1250'
-    lines = [line for line in text.splitlines() if not line.startswith(LINEMARK)]
-    body = '\n'.join(lines).lower()
+    raw = load(path)
+    text, encoding = decode(raw)
+    body = '\n'.join(body_lines(text)).lower()
     count = max([int(n) for n in re.findall(r'\(p(\d+)\)', body)] or [0])
     if count != 7 or 'origin (p2) (p3) (p4)' not in body or 'rotate (p6) (p5) (p7)' not in body:
         return None, 'not origin (p2) (p3) (p4) rotate (p6) (p5) (p7)'
@@ -63,26 +55,12 @@ def examine(path, file):
     return {'raw': raw, 'text': text, 'encoding': encoding, 'header': header, 'family': family, 'hand': hand, 'side': side}, ''
 
 
-def rewrite(info):
-    text = info['text']
-    eol = '\r\n' if '\r\n' in text else '\n'
-    bom = text.startswith('')
-    if bom:
-        text = text[1:]
-    kept = [line for line in text.split(eol) if not line.startswith(LINEMARK)]
-    block = [LINEMARK + ' ' + line for line in info['header']]
-    out = eol.join(block + kept)
-    return (('' if bom else '') + out).encode(info['encoding'])
-
-
 def main():
-    if len(sys.argv) < 2:
-        print(__doc__)
+    chosen = options(sys.argv, __doc__)
+    if chosen is None:
         return 1
-    data = sys.argv[1]
+    data, writing, target = chosen
     folder = os.path.join(data, FOLDER)
-    write = '--write' in sys.argv
-    pack = sys.argv[sys.argv.index('--zip') + 1] if '--zip' in sys.argv else None
     done = []
     skipped = []
     for file in sorted(os.listdir(folder)):
@@ -98,22 +76,10 @@ def main():
     for file, reason in skipped:
         print('skipped %s: %s' % (file, reason))
     print('%d described, %d skipped' % (len(done), len(skipped)))
-    if write:
-        backup = os.path.join(data, 'editor_backup')
-        os.makedirs(backup, exist_ok=True)
-        archive = os.path.join(backup, 'drives_%s.tar' % time.strftime('%Y%m%d_%H%M%S'))
-        with tarfile.open(archive, 'w') as tar:
-            for file, _ in done:
-                tar.add(os.path.join(folder, file), arcname=FOLDER + '/' + file)
-        for file, info in done:
-            with open(os.path.join(folder, file), 'wb') as out:
-                out.write(rewrite(info))
-        print('written, originals in ' + archive)
-    if pack:
-        with zipfile.ZipFile(pack, 'w', zipfile.ZIP_DEFLATED) as out:
-            for file, info in done:
-                out.writestr(FOLDER + '/' + file, rewrite(info))
-        print('packed into ' + pack)
+    if writing:
+        write(data, FOLDER, 'drives', done)
+    if target:
+        pack(target, FOLDER, done)
     return 0
 
 

tools/editor/editor_headers.py

@@ -0,0 +1,74 @@
+"""Shared parts of the scripts writing //$e descriptions into the templates of the scenery."""
+
+import os
+import tarfile
+import time
+import zipfile
+
+LINEMARK = '//$e'
+BOM = ''
+
+
+def load(path):
+    with open(path, 'rb') as source:  # NOSONAR
+        return source.read()
+
+
+def decode(raw, plain='cp1250'):
+    try:
+        text = raw.decode('utf-8')
+        encoding = 'utf-8' if raw.startswith(b'\xef\xbb\xbf') or max(raw, default=0) > 127 else plain
+    except UnicodeDecodeError:
+        text = raw.decode('cp1250')
+        encoding = 'cp1250'
+    return text, encoding
+
+
+def body_lines(text):
+    return [line for line in text.splitlines() if not line.startswith(LINEMARK)]
+
+
+def rewrite(info):
+    if info.get('header') is None:
+        return info['raw']
+    text = info['text']
+    eol = '\r\n' if '\r\n' in text else '\n'
+    bom = text.startswith(BOM)
+    if bom:
+        text = text[1:]
+    kept = [line for line in text.split(eol) if not line.startswith(LINEMARK)]
+    block = [LINEMARK + ' ' + line for line in info['header']]
+    out = eol.join(block + kept)
+    return ((BOM if bom else '') + out).encode(info['encoding'])
+
+
+def options(argv, usage):
+    if len(argv) < 2:
+        print(usage)
+        return None
+    pack = argv[argv.index('--zip') + 1] if '--zip' in argv else None
+    return argv[1], '--write' in argv, pack
+
+
+def write(data, folder, name, entries):
+    backup = os.path.join(data, 'editor_backup')
+    os.makedirs(backup, exist_ok=True)  # NOSONAR
+    archive = os.path.join(backup, time.strftime(name + '_%Y%m%d_%H%M%S.tar'))
+    root = os.path.join(data, folder)
+    with tarfile.open(archive, 'w') as tar:  # NOSONAR
+        for file, _ in entries:
+            tar.add(os.path.join(root, file), arcname=folder + '/' + file)  # NOSONAR
+    for file, info in entries:
+        content = rewrite(info)
+        if content != info['raw']:
+            with open(os.path.join(root, file), 'wb') as out:  # NOSONAR
+                out.write(content)
+    print('written, originals in', archive)
+
+
+def pack(target, folder, entries):
+    with zipfile.ZipFile(target, 'w', zipfile.ZIP_DEFLATED) as out:  # NOSONAR
+        for file, info in entries:
+            out.writestr(folder + '/' + file, rewrite(info))
+    print('packed into ' + target)
+

tools/editor/signal_headers.py

@@ -8,8 +8,8 @@ without --write it only reports what it would write
 import os
 import re
 import sys
-import tarfile
-import time
+
+from editor_headers import body_lines, decode, load, write
 
 FAMILIES = [
     (re.compile(r'^ss[dow]?\d'), 'main'),
@@ -24,8 +24,6 @@ FAMILIES = [
 
 COLOURS = [('zielon', 'z'), ('czerwon', 'c'), ('pomara', 'p'), ('żółt', 'p'), ('biał', 'b'), ('mleczn', 'b'), ('niebiesk', 'n'), ('ślep', 'x'), ('brak', 'x')]
 
-LINEMARK = '//$e'
-
 
 def tokens(text):
     text = re.sub(r'//[^\n]*', ' ', text)
@@ -50,14 +48,9 @@ def statements(words, keyword):
 
 
 def examine(path, file):
-    raw = open(path, 'rb').read()  # NOSONAR
-    try:
-        text = raw.decode('utf-8')
-        encoding = 'utf-8'
-    except UnicodeDecodeError:
-        text = raw.decode('cp1250')
-        encoding = 'cp1250'
-    lines = [line for line in text.splitlines() if not line.startswith(LINEMARK)]
+    raw = load(path)
+    text, encoding = decode(raw, 'utf-8')
+    lines = body_lines(text)
     body = '\n'.join(lines)
     words = tokens(body)
     count = max([int(m) for m in re.findall(r'\(p(\d+)\)', body)] or [0])
@@ -176,24 +169,12 @@ def examine(path, file):
     return {'raw': raw, 'text': text, 'encoding': encoding, 'header': header, 'kind': kind, 'model': model, 'read': read, 'lamps': lamps, 'mount': mount, 'lean': lean, 'count': count}, ''
 
 
-def rewrite(info):
-    text = info['text']
-    eol = '\r\n' if '\r\n' in text else '\n'
-    bom = text.startswith('')
-    if bom:
-        text = text[1:]
-    kept = [line for line in text.split(eol) if not line.startswith(LINEMARK)]
-    block = [LINEMARK + ' ' + line for line in info['header']]
-    out = eol.join(block + kept)
-    return (('' if bom else '') + out).encode(info['encoding'])
-
-
 def main():
     if len(sys.argv) < 2:
         print(__doc__)
         return 1
     root = os.path.join(sys.argv[1], 'scenery')
-    write = '--write' in sys.argv
+    writing = '--write' in sys.argv
     files = sorted(f for f in os.listdir(root) if f.endswith('.inc'))
     files += ['pkp/semafory/' + f for f in sorted(os.listdir(os.path.join(root, 'pkp', 'semafory'))) if f.endswith('.inc')]
     done = []
@@ -217,19 +198,8 @@ def main():
         kinds[info['kind']] = kinds.get(info['kind'], 0) + 1
     print('described: %d %s, skipped: %d' % (len(done), kinds, len(skipped)))
 
-    if write:
-        backup = os.path.join(sys.argv[1], 'editor_backup')
-        os.makedirs(backup, exist_ok=True)  # NOSONAR
-        archive = os.path.join(backup, time.strftime('signals_%Y%m%d_%H%M%S.tar'))
-        with tarfile.open(archive, 'w') as tar:  # NOSONAR
-            for file, _ in done:
-                tar.add(os.path.join(root, file), arcname='scenery/' + file)  # NOSONAR
-        for file, info in done:
-            content = rewrite(info)
-            if content != info['raw']:
-                with open(os.path.join(root, file), 'wb') as out:  # NOSONAR
-                    out.write(content)
-        print('written, originals in', archive)
+    if writing:
+        write(sys.argv[1], 'scenery', 'signals', done)
     return 0
 
 

tools/editor/turntable_headers.py

@@ -8,35 +8,23 @@ without --write it only reports what it would write; --zip packs every described
 import os
 import re
 import sys
-import tarfile
-import time
-import zipfile
 
-LINEMARK = '//$e'
+from editor_headers import LINEMARK, body_lines, decode, load, options, pack, write
+
 FOLDER = 'scenery'
 ROTATION = re.compile(r'^event\s+\S*tor(\d+)rot\s+animation\s+\S+\s+\S+\s+rotate\s+\S+\s+\S+\s+\S+\s+(\S+)', re.I | re.M)
 MODEL = re.compile(r'model\s+0\s+0\s+0\s+0\s+\S*?obrotnica(\d+)m\.t3d\s+\(p6\)', re.I)
 
 
-def decode(raw):
-    try:
-        text = raw.decode('utf-8')
-        encoding = 'utf-8' if raw.startswith(b'\xef\xbb\xbf') or max(raw, default=0) > 127 else 'cp1250'
-    except UnicodeDecodeError:
-        text = raw.decode('cp1250')
-        encoding = 'cp1250'
-    return text, encoding
-
-
 def examine(path, relative):
-    raw = open(path, 'rb').read()  # NOSONAR
+    raw = load(path)
     if re.search(rb'tor\d+rot\s+animation', raw, re.I) is None:
         return None, None
     text, encoding = decode(raw)
     marks = [line for line in text.splitlines() if line.startswith(LINEMARK)]
     if any('category: turntable' in line for line in marks):
         return {'raw': raw, 'header': None, 'described': True}, ''
-    body = '\n'.join(line for line in text.splitlines() if not line.startswith(LINEMARK))
+    body = '\n'.join(body_lines(text))
     lower = body.lower()
     if 'origin (p2) (p3) (p4)' not in lower or 'rotate 0 (p5) 0' not in lower:
         return None, 'not origin (p2) (p3) (p4) rotate 0 (p5) 0'
@@ -75,28 +63,12 @@ def examine(path, relative):
     return {'raw': raw, 'text': text, 'encoding': encoding, 'header': header, 'described': False, 'length': length, 'angles': angles}, ''
 
 
-def rewrite(info):
-    if info['header'] is None:
-        return info['raw']
-    text = info['text']
-    eol = '\r\n' if '\r\n' in text else '\n'
-    bom = text.startswith('')
-    if bom:
-        text = text[1:]
-    kept = [line for line in text.split(eol) if not line.startswith(LINEMARK)]
-    block = [LINEMARK + ' ' + line for line in info['header']]
-    out = eol.join(block + kept)
-    return (('' if bom else '') + out).encode(info['encoding'])
-
-
 def main():
-    if len(sys.argv) < 2:
-        print(__doc__)
+    chosen = options(sys.argv, __doc__)
+    if chosen is None:
         return 1
-    data = sys.argv[1]
+    data, writing, target = chosen
     folder = os.path.join(data, FOLDER)
-    write = '--write' in sys.argv
-    pack = sys.argv[sys.argv.index('--zip') + 1] if '--zip' in sys.argv else None
     done = []
     skipped = []
     for root, _, files in os.walk(folder):
@@ -119,22 +91,10 @@ def main():
         print('skipped %s: %s' % (file, reason))
     fresh = [(file, info) for file, info in done if not info['described']]
     print('%d described now, %d already described, %d skipped' % (len(fresh), len(done) - len(fresh), len(skipped)))
-    if write and fresh:
-        backup = os.path.join(data, 'editor_backup')
-        os.makedirs(backup, exist_ok=True)
-        archive = os.path.join(backup, 'turntables_%s.tar' % time.strftime('%Y%m%d_%H%M%S'))
-        with tarfile.open(archive, 'w') as tar:
-            for file, _ in fresh:
-                tar.add(os.path.join(folder, file), arcname=FOLDER + '/' + file)
-        for file, info in fresh:
-            with open(os.path.join(folder, file), 'wb') as out:
-                out.write(rewrite(info))
-        print('written, originals in ' + archive)
-    if pack:
-        with zipfile.ZipFile(pack, 'w', zipfile.ZIP_DEFLATED) as out:
-            for file, info in done:
-                out.writestr(FOLDER + '/' + file, rewrite(info))
-        print('packed into ' + pack)
+    if writing and fresh:
+        write(data, FOLDER, 'turntables', fresh)
+    if target:
+        pack(target, FOLDER, done)
     return 0