From 4695b318cda72fda4fde54bc86aa37a7154547e4 Mon Sep 17 00:00:00 2001 From: ppcvote Date: Tue, 1 Sep 2026 03:12:02 +0800 Subject: [PATCH] Write generated data files as UTF-8 with LF instead of the locale encoding doctools opened every generated artifact in text mode with no encoding, so the descriptions' curly quotes were written in the host code page. Those code points exist in cp950 and cp1252, so the write succeeded and emitted non-UTF-8 bytes with no warning. The CSV writer also lacked newline="", turning to_csv's line endings into \r\r\n. Running the pre-commit hook on a cp950 host modified 185 files under data/, 34 of which stopped being valid UTF-8. With this change the same run leaves data/ byte-identical. --- src/ssvc/decision_tables/helpers.py | 6 ++- src/ssvc/doctools.py | 12 ++--- src/ssvc/md_gen.py | 4 +- src/test/test_doctools_encoding.py | 77 +++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 9 deletions(-) create mode 100644 src/test/test_doctools_encoding.py diff --git a/src/ssvc/decision_tables/helpers.py b/src/ssvc/decision_tables/helpers.py index 1700418cc..21e70d1d9 100644 --- a/src/ssvc/decision_tables/helpers.py +++ b/src/ssvc/decision_tables/helpers.py @@ -62,9 +62,11 @@ def write_csv( csv_path = os.path.join(target_dir, csvfile) - with open(csv_path, "w") as fp: + with open(csv_path, "w", encoding="utf-8", newline="") as fp: fp.write( - decision_table_to_longform_df(decision_table).to_csv(index=index) + decision_table_to_longform_df(decision_table).to_csv( + index=index, lineterminator="\n" + ) ) diff --git a/src/ssvc/doctools.py b/src/ssvc/doctools.py index da652bc51..73a9f9067 100755 --- a/src/ssvc/doctools.py +++ b/src/ssvc/doctools.py @@ -181,7 +181,7 @@ def dump_json( with EnsureDirExists(dirname): try: logger.info(f"Writing {json_file}") - with open(json_file, "x") as f: + with open(json_file, "x", encoding="utf-8", newline="\n") as f: f.write(dp.model_dump_json(indent=2)) f.write("\n") # newline at end of file except FileExistsError: @@ -194,7 +194,7 @@ def dump_json( def dump_schema(filepath: str, schema: dict) -> None: schema = order_schema(schema) logger.info(f"Writing schema to {filepath}") - with open(filepath, "w") as f: + with open(filepath, "w", encoding="utf-8", newline="\n") as f: json.dump(schema, f, indent=2) f.write("\n") @@ -244,7 +244,7 @@ def dump_decision_table( with EnsureDirExists(dirname): try: logger.info(f"Writing {json_file}") - with open(json_file, "x") as f: + with open(json_file, "x", encoding="utf-8", newline="\n") as f: f.write(dt.model_dump_json(indent=2)) f.write("\n") # newline at end of file except FileExistsError: @@ -270,11 +270,11 @@ def dump_decision_table_csv( with EnsureDirExists(dirname): try: logger.info("Writing {csv_file}") - with open(csv_file, "x") as f: + with open(csv_file, "x", encoding="utf-8", newline="") as f: df = decision_table_to_longform_df(dt=dt) # set the index title df.index.name = "row" - f.write(df.to_csv(index=True)) + f.write(df.to_csv(index=True, lineterminator="\n")) except FileExistsError: logger.warning( f"File {csv_file} already exists, use --overwrite to replace" @@ -342,7 +342,7 @@ def main(): with EnsureDirExists(jsondir): try: logger.info(f"Writing {registry_json}") - with open(registry_json, "x") as f: + with open(registry_json, "x", encoding="utf-8", newline="\n") as f: f.write(registry.model_dump_json(indent=2, exclude_none=True)) f.write("\n") # newline at end of file except FileExistsError: diff --git a/src/ssvc/md_gen.py b/src/ssvc/md_gen.py index 51707bbbc..c02119a9d 100644 --- a/src/ssvc/md_gen.py +++ b/src/ssvc/md_gen.py @@ -141,7 +141,9 @@ def main(): print(f"Module {module} does not exist") continue - with open(os.path.join(md_dir, fname), "w") as f: + with open( + os.path.join(md_dir, fname), "w", encoding="utf-8", newline="\n" + ) as f: f.write( PAGE_TOP_TEMPLATE.format( dp_name=snake_to_title(dp_fname), module=dp_fname diff --git a/src/test/test_doctools_encoding.py b/src/test/test_doctools_encoding.py new file mode 100644 index 000000000..51dbbc400 --- /dev/null +++ b/src/test/test_doctools_encoding.py @@ -0,0 +1,77 @@ +# Copyright (c) 2023-2026 Carnegie Mellon University. +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE +# ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. +# CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT +# NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR +# MERCHANTABILITY, EXCLUSIVITY, OR RESULTS OBTAINED FROM USE +# OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE +# ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM +# PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT. +# Licensed under a MIT (SEI)-style license, please see LICENSE or contact +# permission@sei.cmu.edu for full terms. +# [DISTRIBUTION STATEMENT A] This material has been approved for +# public release and unlimited distribution. Please see Copyright notice +# for non-US Government use and distribution. +# This Software includes and/or makes use of Third-Party Software each +# subject to its own license. +# DM24-0278 +"""The generated data files must not depend on the machine that wrote them. + +The decision point descriptions carry curly quotes. Those code points also exist +in the legacy Windows code pages, so writing them without an explicit encoding +succeeds and silently emits non-UTF-8 bytes: no exception, no warning, exit 0. +The pre-commit hook and run_doctools.yml both tell a contributor to regenerate +and push the result, so the corruption reaches the index. +""" +import os +import pathlib +import subprocess +import sys +import tempfile +import unittest + +DOCTOOLS = pathlib.Path(__file__).resolve().parents[1] / "ssvc" / "doctools.py" + + +class DoctoolsEncodingTest(unittest.TestCase): + def test_generation_never_falls_back_to_the_locale(self): + """No file written by a full run may be opened without an encoding. + + Run under -X warn_default_encoding (PEP 597), which reports an implicit + encoding on every platform. Asserting on the generated bytes instead + would pass on a UTF-8 runner whether or not the defect is present, + which is why CI has never seen this. + + Only warnings raised from inside this package count: dependencies open + files without an encoding at import time and would otherwise decide the + result. + """ + with tempfile.TemporaryDirectory() as datadir: + env = dict(os.environ, PYTHONWARNDEFAULTENCODING="1") + result = subprocess.run( + [ + sys.executable, + "-X", + "warn_default_encoding", + str(DOCTOOLS), + f"--datadir={datadir}", + "--overwrite", + ], + capture_output=True, + env=env, + timeout=600, + ) + + stderr = result.stderr.decode("utf-8", "replace") + ours = [ + line + for line in stderr.splitlines() + if "EncodingWarning" in line and f"{os.sep}ssvc{os.sep}" in line + ] + self.assertEqual(result.returncode, 0, stderr) + self.assertEqual(ours, [], "\n".join(ours)) + + +if __name__ == "__main__": + unittest.main()