Merge pull request #3367 from citrus-it/pyc

Explicitly set the embedded timestamp in .pyc files

build/python311/patches/pyc-timestamp.patch

@@ -0,0 +1,26 @@
+
+When packaging python modules along with their compiled source, we
+deliberately set the timestamp attributes for both the source
+and compiled file so that they are not recompiled the first time
+root uses them (via pkg, for example). Doing so causes `pkg validate`
+errors and slows down the initial invocation.
+
+However, the timestamp and size of the .py file is also embedded within
+the .pyc header. This patch allows overriding that embedded timestamp
+with a value of our choosing. See lib/functions.sh for how this is set.
+
+diff -wpruN --no-dereference '--exclude=*.orig' a~/Lib/py_compile.py a/Lib/py_compile.py
+--- a~/Lib/py_compile.py	1970-01-01 00:00:00
++++ a/Lib/py_compile.py	1970-01-01 00:00:00
+@@ -159,6 +159,11 @@ def compile(file, cfile=None, dfile=None
+         pass
+     if invalidation_mode == PycInvalidationMode.TIMESTAMP:
+         source_stats = loader.path_stats(file)
++        if fpepoch := os.environ.get('FORCE_PYC_TIMESTAMP'):
++            try:
++                source_stats['mtime'] = int(fpepoch)
++            except ValueError:
++                raise ValueError("FORCE_PYC_TIMESTAMP is not a valid integer")
+         bytecode = importlib._bootstrap_external._code_to_timestamp_pyc(
+             code, source_stats['mtime'], source_stats['size'])
+     else:

build/python311/patches/series

@@ -10,6 +10,7 @@ encoding-alias.patch
 locale-encoding.patch
 cgiserver.patch
 static-assert.patch
+pyc-timestamp.patch
 #
 # Additional modules
 module-ucred.patch

lib/functions.sh

@@ -1062,10 +1062,21 @@ prep_build() {
 
     # Generate timestamps
     typeset now=`TZ=UTC $DATE +%s`
+    typeset TS_SRC_EPOCH=$((now - 60))
+    typeset TS_OBJ_EPOCH=$((now - 30))
     typeset TS_FMT="%Y%m%dT%H%M%SZ"
-    typeset TS_SRC=`$DATE -r $((now - 60)) +$TS_FMT`
-    typeset TS_OBJ=`$DATE -r $((now - 30)) +$TS_FMT`
-
+    typeset TS_SRC=`$DATE -r $TS_SRC_EPOCH +$TS_FMT`
+    typeset TS_OBJ=`$DATE -r $TS_OBJ_EPOCH +$TS_FMT`
+
+    # Python is patched to use the value of this variable as the timestamp that
+    # it embeds in .pyc files. We need to make sure that this embedded
+    # timestamp matches the timestamp that the packaging system will apply to
+    # the corresponding source .py file.
+    export FORCE_PYC_TIMESTAMP=$TS_SRC_EPOCH
+
+    # These tokens are used by rules in lib/mog/global-transforms.mog to
+    # automatically apply timestamp attributes to python modules and their
+    # compiled form. They can also be used by other packages in their local.mog
     SYS_XFORM_ARGS+=" -DTS_SRC=$TS_SRC -DTS_OBJ=$TS_OBJ"
 
     logmsg "--- Creating temporary installation directory"

lib/mog/global-transforms.mog

@@ -47,13 +47,12 @@
 <transform file -> default mode 0644>
 <transform dir -> default mode 0755>
 
-# The timestamp on any compiled python objects that are shipped in a package
-# must always be newer than the corresponding source file, or python recompiles
-# and the benefit of shipping the compiled version is lost, particularly if
-# python is not running with privileges that can update the compiled version on
-# disk.
+# The timestamp of Python .py files is embedded in the associated compiled .pyc
+# file. We must explicitly set the timstamp for these otherwise Python will
+# recompile and the benefit of shipping the compiled version is lost,
+# particularly if python is not running with privileges that can update the
+# compiled version on disk.
 <transform file path=.*\.py$ -> default timestamp $(TS_SRC)>
-<transform file path=.*\.pyc$ -> default timestamp $(TS_OBJ)>
 
 # Overrides
 <transform file dir path=(lib|var)/svc/manifest/ -> set group sys>