Write generated data files as UTF-8 with LF instead of the locale encoding

src/ssvc/decision_tables/helpers.py

@@ -62,9 +62,11 @@ def write_csv(
 
     csv_path = os.path.join(target_dir, csvfile)
 
-    with open(csv_path, "w") as fp:
+    with open(csv_path, "w", encoding="utf-8", newline="") as fp:
         fp.write(
-            decision_table_to_longform_df(decision_table).to_csv(index=index)
+            decision_table_to_longform_df(decision_table).to_csv(
+                index=index, lineterminator="\n"
+            )
         )
 
 

src/ssvc/doctools.py

@@ -181,7 +181,7 @@ def dump_json(
     with EnsureDirExists(dirname):
         try:
             logger.info(f"Writing {json_file}")
-            with open(json_file, "x") as f:
+            with open(json_file, "x", encoding="utf-8", newline="\n") as f:
                 f.write(dp.model_dump_json(indent=2))
                 f.write("\n")  # newline at end of file
         except FileExistsError:
@@ -194,7 +194,7 @@ def dump_json(
 def dump_schema(filepath: str, schema: dict) -> None:
     schema = order_schema(schema)
     logger.info(f"Writing schema to {filepath}")
-    with open(filepath, "w") as f:
+    with open(filepath, "w", encoding="utf-8", newline="\n") as f:
         json.dump(schema, f, indent=2)
         f.write("\n")
 
@@ -244,7 +244,7 @@ def dump_decision_table(
     with EnsureDirExists(dirname):
         try:
             logger.info(f"Writing {json_file}")
-            with open(json_file, "x") as f:
+            with open(json_file, "x", encoding="utf-8", newline="\n") as f:
                 f.write(dt.model_dump_json(indent=2))
                 f.write("\n")  # newline at end of file
         except FileExistsError:
@@ -270,11 +270,11 @@ def dump_decision_table_csv(
     with EnsureDirExists(dirname):
         try:
             logger.info("Writing {csv_file}")
-            with open(csv_file, "x") as f:
+            with open(csv_file, "x", encoding="utf-8", newline="") as f:
                 df = decision_table_to_longform_df(dt=dt)
                 # set the index title
                 df.index.name = "row"
-                f.write(df.to_csv(index=True))
+                f.write(df.to_csv(index=True, lineterminator="\n"))
         except FileExistsError:
             logger.warning(
                 f"File {csv_file} already exists, use --overwrite to replace"
@@ -342,7 +342,7 @@ def main():
     with EnsureDirExists(jsondir):
         try:
             logger.info(f"Writing {registry_json}")
-            with open(registry_json, "x") as f:
+            with open(registry_json, "x", encoding="utf-8", newline="\n") as f:
                 f.write(registry.model_dump_json(indent=2, exclude_none=True))
                 f.write("\n")  # newline at end of file
         except FileExistsError:

src/ssvc/md_gen.py

@@ -141,7 +141,9 @@ def main():
             print(f"Module {module} does not exist")
             continue
 
-        with open(os.path.join(md_dir, fname), "w") as f:
+        with open(
+            os.path.join(md_dir, fname), "w", encoding="utf-8", newline="\n"
+        ) as f:
             f.write(
                 PAGE_TOP_TEMPLATE.format(
                     dp_name=snake_to_title(dp_fname), module=dp_fname

src/test/test_doctools_encoding.py

@@ -0,0 +1,77 @@
+#  Copyright (c) 2023-2026 Carnegie Mellon University.
+#  NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE
+#  ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS.
+#  CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY KIND,
+#  EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT
+#  NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR
+#  MERCHANTABILITY, EXCLUSIVITY, OR RESULTS OBTAINED FROM USE
+#  OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE
+#  ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM
+#  PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT.
+#  Licensed under a MIT (SEI)-style license, please see LICENSE or contact
+#  permission@sei.cmu.edu for full terms.
+#  [DISTRIBUTION STATEMENT A] This material has been approved for
+#  public release and unlimited distribution. Please see Copyright notice
+#  for non-US Government use and distribution.
+#  This Software includes and/or makes use of Third-Party Software each
+#  subject to its own license.
+#  DM24-0278
+"""The generated data files must not depend on the machine that wrote them.
+
+The decision point descriptions carry curly quotes. Those code points also exist
+in the legacy Windows code pages, so writing them without an explicit encoding
+succeeds and silently emits non-UTF-8 bytes: no exception, no warning, exit 0.
+The pre-commit hook and run_doctools.yml both tell a contributor to regenerate
+and push the result, so the corruption reaches the index.
+"""
+import os
+import pathlib
+import subprocess
+import sys
+import tempfile
+import unittest
+
+DOCTOOLS = pathlib.Path(__file__).resolve().parents[1] / "ssvc" / "doctools.py"
+
+
+class DoctoolsEncodingTest(unittest.TestCase):
+    def test_generation_never_falls_back_to_the_locale(self):
+        """No file written by a full run may be opened without an encoding.
+
+        Run under -X warn_default_encoding (PEP 597), which reports an implicit
+        encoding on every platform. Asserting on the generated bytes instead
+        would pass on a UTF-8 runner whether or not the defect is present,
+        which is why CI has never seen this.
+
+        Only warnings raised from inside this package count: dependencies open
+        files without an encoding at import time and would otherwise decide the
+        result.
+        """
+        with tempfile.TemporaryDirectory() as datadir:
+            env = dict(os.environ, PYTHONWARNDEFAULTENCODING="1")
+            result = subprocess.run(
+                [
+                    sys.executable,
+                    "-X",
+                    "warn_default_encoding",
+                    str(DOCTOOLS),
+                    f"--datadir={datadir}",
+                    "--overwrite",
+                ],
+                capture_output=True,
+                env=env,
+                timeout=600,
+            )
+
+        stderr = result.stderr.decode("utf-8", "replace")
+        ours = [
+            line
+            for line in stderr.splitlines()
+            if "EncodingWarning" in line and f"{os.sep}ssvc{os.sep}" in line
+        ]
+        self.assertEqual(result.returncode, 0, stderr)
+        self.assertEqual(ours, [], "\n".join(ours))
+
+
+if __name__ == "__main__":
+    unittest.main()