diff --git a/src/ssvc/decision_tables/helpers.py b/src/ssvc/decision_tables/helpers.py index 1700418c..21e70d1d 100644 --- a/src/ssvc/decision_tables/helpers.py +++ b/src/ssvc/decision_tables/helpers.py @@ -62,9 +62,11 @@ def write_csv( csv_path = os.path.join(target_dir, csvfile) - with open(csv_path, "w") as fp: + with open(csv_path, "w", encoding="utf-8", newline="") as fp: fp.write( - decision_table_to_longform_df(decision_table).to_csv(index=index) + decision_table_to_longform_df(decision_table).to_csv( + index=index, lineterminator="\n" + ) ) diff --git a/src/ssvc/doctools.py b/src/ssvc/doctools.py index da652bc5..73a9f906 100755 --- a/src/ssvc/doctools.py +++ b/src/ssvc/doctools.py @@ -181,7 +181,7 @@ def dump_json( with EnsureDirExists(dirname): try: logger.info(f"Writing {json_file}") - with open(json_file, "x") as f: + with open(json_file, "x", encoding="utf-8", newline="\n") as f: f.write(dp.model_dump_json(indent=2)) f.write("\n") # newline at end of file except FileExistsError: @@ -194,7 +194,7 @@ def dump_json( def dump_schema(filepath: str, schema: dict) -> None: schema = order_schema(schema) logger.info(f"Writing schema to {filepath}") - with open(filepath, "w") as f: + with open(filepath, "w", encoding="utf-8", newline="\n") as f: json.dump(schema, f, indent=2) f.write("\n") @@ -244,7 +244,7 @@ def dump_decision_table( with EnsureDirExists(dirname): try: logger.info(f"Writing {json_file}") - with open(json_file, "x") as f: + with open(json_file, "x", encoding="utf-8", newline="\n") as f: f.write(dt.model_dump_json(indent=2)) f.write("\n") # newline at end of file except FileExistsError: @@ -270,11 +270,11 @@ def dump_decision_table_csv( with EnsureDirExists(dirname): try: logger.info("Writing {csv_file}") - with open(csv_file, "x") as f: + with open(csv_file, "x", encoding="utf-8", newline="") as f: df = decision_table_to_longform_df(dt=dt) # set the index title df.index.name = "row" - f.write(df.to_csv(index=True)) + f.write(df.to_csv(index=True, lineterminator="\n")) except FileExistsError: logger.warning( f"File {csv_file} already exists, use --overwrite to replace" @@ -342,7 +342,7 @@ def main(): with EnsureDirExists(jsondir): try: logger.info(f"Writing {registry_json}") - with open(registry_json, "x") as f: + with open(registry_json, "x", encoding="utf-8", newline="\n") as f: f.write(registry.model_dump_json(indent=2, exclude_none=True)) f.write("\n") # newline at end of file except FileExistsError: diff --git a/src/ssvc/md_gen.py b/src/ssvc/md_gen.py index 51707bbb..c02119a9 100644 --- a/src/ssvc/md_gen.py +++ b/src/ssvc/md_gen.py @@ -141,7 +141,9 @@ def main(): print(f"Module {module} does not exist") continue - with open(os.path.join(md_dir, fname), "w") as f: + with open( + os.path.join(md_dir, fname), "w", encoding="utf-8", newline="\n" + ) as f: f.write( PAGE_TOP_TEMPLATE.format( dp_name=snake_to_title(dp_fname), module=dp_fname diff --git a/src/test/test_doctools_encoding.py b/src/test/test_doctools_encoding.py new file mode 100644 index 00000000..51dbbc40 --- /dev/null +++ b/src/test/test_doctools_encoding.py @@ -0,0 +1,77 @@ +# Copyright (c) 2023-2026 Carnegie Mellon University. +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE +# ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. +# CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT +# NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR +# MERCHANTABILITY, EXCLUSIVITY, OR RESULTS OBTAINED FROM USE +# OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE +# ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM +# PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT. +# Licensed under a MIT (SEI)-style license, please see LICENSE or contact +# permission@sei.cmu.edu for full terms. +# [DISTRIBUTION STATEMENT A] This material has been approved for +# public release and unlimited distribution. Please see Copyright notice +# for non-US Government use and distribution. +# This Software includes and/or makes use of Third-Party Software each +# subject to its own license. +# DM24-0278 +"""The generated data files must not depend on the machine that wrote them. + +The decision point descriptions carry curly quotes. Those code points also exist +in the legacy Windows code pages, so writing them without an explicit encoding +succeeds and silently emits non-UTF-8 bytes: no exception, no warning, exit 0. +The pre-commit hook and run_doctools.yml both tell a contributor to regenerate +and push the result, so the corruption reaches the index. +""" +import os +import pathlib +import subprocess +import sys +import tempfile +import unittest + +DOCTOOLS = pathlib.Path(__file__).resolve().parents[1] / "ssvc" / "doctools.py" + + +class DoctoolsEncodingTest(unittest.TestCase): + def test_generation_never_falls_back_to_the_locale(self): + """No file written by a full run may be opened without an encoding. + + Run under -X warn_default_encoding (PEP 597), which reports an implicit + encoding on every platform. Asserting on the generated bytes instead + would pass on a UTF-8 runner whether or not the defect is present, + which is why CI has never seen this. + + Only warnings raised from inside this package count: dependencies open + files without an encoding at import time and would otherwise decide the + result. + """ + with tempfile.TemporaryDirectory() as datadir: + env = dict(os.environ, PYTHONWARNDEFAULTENCODING="1") + result = subprocess.run( + [ + sys.executable, + "-X", + "warn_default_encoding", + str(DOCTOOLS), + f"--datadir={datadir}", + "--overwrite", + ], + capture_output=True, + env=env, + timeout=600, + ) + + stderr = result.stderr.decode("utf-8", "replace") + ours = [ + line + for line in stderr.splitlines() + if "EncodingWarning" in line and f"{os.sep}ssvc{os.sep}" in line + ] + self.assertEqual(result.returncode, 0, stderr) + self.assertEqual(ours, [], "\n".join(ours)) + + +if __name__ == "__main__": + unittest.main()