Merge pull request #31 from lsst/tickets/DM-39756

DM-39756: Switch from pkg_resources to importlib.resources

.github/workflows/build.yml

@@ -6,17 +6,25 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.10", "3.11"]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Build and install wheel
       run: |
         pip install virtualenv
         test/test_wheel_install.sh
-    # TODO: Add lint and test steps once we actually pass these tests. Crib from
-    # https://github.com/lsst/daf_butler/blob/main/.github/workflows/lint.yaml.
+    - name: Install pytest and fastavro (needed by tests)
+      run: |
+        pip install --upgrade pip
+        pip install pytest fastavro
+    - name: Ensure we have a usable version installed
+      run: |
+        pip install -e .
+    - name: run tests
+      run: |
+        pytest test

.github/workflows/lint.yaml

@@ -0,0 +1,11 @@
+name: lint
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  call-workflow:
+    uses: lsst/rubin_workflows/.github/workflows/lint.yaml@main

python/lsst/alert/packet/bin/simulateAlerts.py

@@ -23,7 +23,7 @@
 
 import argparse
 
-import fastavro
+import fastavro  # noqa: F401
 
 import lsst.alert.packet
 
@@ -48,22 +48,22 @@ def main():
                   'prvDiaForcedSources': args.visits_per_year//12,
                   'prvDiaNondetectionLimits': 0}
     alerts = [lsst.alert.packet.simulate_alert(schema.definition,
-                                        keepNull=['ssObject'],
-                                        arrayCount=arrayCount)
+                                               keepNull=['ssObject'],
+                                               arrayCount=arrayCount)
               for _ in range(args.num_alerts)]
 
     for alert in alerts:
-        assert(schema.validate(alert))
+        assert schema.validate(alert)
 
     with open(args.output_filename, "wb") as f:
         schema.store_alerts(f, alerts)
 
     with open(args.output_filename, "rb") as f:
         writer_schema, loaded_alerts = schema.retrieve_alerts(f)
 
-    assert(schema == writer_schema)
+    assert schema == writer_schema
     for a1, a2 in zip(alerts, loaded_alerts):
-        assert(a1 == a2)
+        assert a1 == a2
 
 
 if __name__ == '__main__':

python/lsst/alert/packet/bin/validateAvroRoundTrip.py

@@ -30,7 +30,6 @@
 import filecmp
 import json
 import os
-import sys
 import tempfile
 
 import lsst.alert.packet
@@ -59,6 +58,7 @@ def check_file_round_trip(baseline, received_data):
             f.write(received_data)
         assert filecmp.cmp(baseline, filename, shallow=False)
 
+
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument('--schema-version', type=str,
@@ -75,24 +75,25 @@ def parse_args():
 
     return parser.parse_args()
 
+
 def main():
     args = parse_args()
     if args.schema_version == "latest":
         schema_major, schema_minor = lsst.alert.packet.get_latest_schema_version()
     else:
         schema_major, schema_minor = args.schema_version.split(".")
-    schema_root = lsst.alert.packet.get_schema_path(schema_major, schema_minor)
-
-    alert_schema = lsst.alert.packet.Schema.from_file(
-        os.path.join(schema_root,
-                     schema_filename(schema_major, schema_minor)),
-    )
-    if args.input_data:
-        input_data = args.input_data
-    else:
-        input_data = os.path.join(schema_root, "sample_data", SAMPLE_FILENAME)
-    with open(input_data) as f:
-        json_data = json.load(f)
+
+    with lsst.alert.packet.get_schema_path(schema_major, schema_minor) as schema_root:
+        alert_schema = lsst.alert.packet.Schema.from_file(
+            os.path.join(schema_root,
+                         schema_filename(schema_major, schema_minor)),
+        )
+        if args.input_data:
+            input_data = args.input_data
+        else:
+            input_data = os.path.join(schema_root, "sample_data", SAMPLE_FILENAME)
+        with open(input_data) as f:
+            json_data = json.load(f)
 
     # Load difference stamp if included
     stamp_size = 0

python/lsst/alert/packet/io.py

@@ -30,6 +30,7 @@
 
 __all__ = ["load_stamp", "retrieve_alerts"]
 
+
 def load_stamp(file_path):
     """Load a cutout postage stamp file to include in alert.
     """
@@ -39,6 +40,7 @@ def load_stamp(file_path):
         cutout_dict = {"fileName": fileoutname, "stampData": cutout_data}
     return cutout_dict
 
+
 def retrieve_alerts(fp, reader_schema=None):
     """Read alert packets from the given I/O stream.
 
@@ -82,7 +84,8 @@ def retrieve_alerts(fp, reader_schema=None):
         first_record = next(reader)
         records = itertools.chain([first_record], reader)
     except StopIteration:
-        # The file has zero records in it. It might still have a schema, though.
+        # The file has zero records in it. It might still have a schema,
+        # though.
         records = []
     writer_schema = Schema(reader.writer_schema)
     return writer_schema, records

python/lsst/alert/packet/schema.py

@@ -22,21 +22,34 @@
 """Routines for working with Avro schemas.
 """
 
+import contextlib
 import io
 import os.path
-import pkg_resources
 from pathlib import PurePath
+from importlib import resources
 
 import fastavro
 
 __all__ = ["get_schema_root", "get_latest_schema_version", "get_schema_path",
            "Schema", "get_path_to_latest_schema"]
 
 
+def _get_ref(*args):
+    """Return the package resource file path object.
+
+    Parameters are relative to lsst.alert.packet.
+    """
+    return resources.files("lsst.alert.packet").joinpath(*args)
+
+
+@contextlib.contextmanager
 def get_schema_root():
     """Return the root of the directory within which schemas are stored.
+
+    Returned as a context manager yielding the path to the root.
     """
-    return pkg_resources.resource_filename(__name__, "schema")
+    with resources.as_file(_get_ref("schema")) as f:
+        yield str(f)
 
 
 def get_latest_schema_version():
@@ -50,12 +63,14 @@ def get_latest_schema_version():
         The minor version number.
 
     """
-    val = pkg_resources.resource_string(__name__, "schema/latest.txt")
+    with _get_ref("schema", "latest.txt").open("rb") as fh:
+        val = fh.read()
     clean = val.strip()
     major, minor = clean.split(b".", 1)
     return int(major), int(minor)
 
 
+@contextlib.contextmanager
 def get_schema_path(major, minor):
     """Get the path to a package resource directory housing alert schema
     definitions.
@@ -73,13 +88,11 @@ def get_schema_path(major, minor):
         Path to the directory containing the schemas.
 
     """
-
-    # Note that as_posix() is right here, since pkg_resources
-    # always uses slash-delimited paths, even on Windows.
-    path = PurePath(f"schema/{major}/{minor}/")
-    return pkg_resources.resource_filename(__name__, path.as_posix())
+    with resources.as_file(_get_ref("schema", str(major), str(minor))) as f:
+        yield str(f)
 
 
+@contextlib.contextmanager
 def get_path_to_latest_schema():
     """Get the path to the primary schema file for the latest schema.
 
@@ -90,8 +103,8 @@ def get_path_to_latest_schema():
     """
 
     major, minor = get_latest_schema_version()
-    schema_path = PurePath(get_schema_path(major, minor))
-    return (schema_path / f"lsst.v{major}_{minor}.alert.avsc").as_posix()
+    with get_schema_path(major, minor) as schema_path:
+        yield (PurePath(schema_path) / f"lsst.v{major}_{minor}.alert.avsc").as_posix()
 
 
 def resolve_schema_definition(to_resolve, seen_names=None):
@@ -294,30 +307,33 @@ def __eq__(self, other):
 
     @classmethod
     def from_file(cls, filename=None):
-        """Instantiate a `Schema` by reading its definition from the filesystem.
+        """Instantiate a `Schema` by reading its definition from the
+        filesystem.
 
         Parameters
         ----------
         filename : `str`, optional
-            Path to the schema root (/path/to/lsst.vM_m.alert.avsc). 
-            Will recursively load referenced schemas, assuming they can be 
+            Path to the schema root (/path/to/lsst.vM_m.alert.avsc).
+            Will recursively load referenced schemas, assuming they can be
             found; otherwise, will raise. If `None` (the
             default), will load the latest schema defined in this package.
         """
         if filename is None:
             major, minor = get_latest_schema_version()
             root_name = f"lsst.v{major}_{minor}.alert"
-            filename = os.path.join(
-                get_schema_path(major, minor),
-                root_name + ".avsc",
-            )
+            with get_schema_path(major, minor) as schema_path:
+                filename = os.path.join(
+                    schema_path,
+                    root_name + ".avsc",
+                )
+                schema_definition = fastavro.schema.load_schema(filename)
         else:
             root_name = PurePath(filename).stem
+            schema_definition = fastavro.schema.load_schema(filename)
 
-        schema_definition = fastavro.schema.load_schema(filename)
         if hasattr(fastavro.schema._schema, 'SCHEMA_DEFS'):
-            # Old fastavro gives a back a list if it recursively loaded more than one
-            # file, otherwise a dict.
+            # Old fastavro gives a back a list if it recursively loaded more
+            # than one file, otherwise a dict.
             if isinstance(schema_definition, dict):
                 schema_definition = [schema_definition]
 

python/lsst/alert/packet/schemaRegistry.py

@@ -137,13 +137,15 @@ def from_filesystem(cls, root=None, schema_root="lsst.v5_0.alert"):
         """
         from .schema import Schema
         from .schema import get_schema_root
-        if not root:
-            root = get_schema_root()
-        registry = cls()
-        schema_root_file = schema_root + ".avsc"
-        for root, dirs, files in os.walk(root, followlinks=False):
-            if schema_root_file in files:
-                schema = Schema.from_file(os.path.join(root, schema_root_file))
-                version = ".".join(root.split("/")[-2:])
-                registry.register_schema(schema, version)
+
+        with get_schema_root() as default_root:
+            if not root:
+                root = default_root
+            registry = cls()
+            schema_root_file = schema_root + ".avsc"
+            for root, dirs, files in os.walk(root, followlinks=False):
+                if schema_root_file in files:
+                    schema = Schema.from_file(os.path.join(root, schema_root_file))
+                    version = ".".join(root.split("/")[-2:])
+                    registry.register_schema(schema, version)
         return registry

python/lsst/alert/packet/simulate.py

@@ -27,36 +27,43 @@
 
 __all__ = ["simulate_alert"]
 
+
 def randomNull():
     """Provide a random value of the Avro `null` type.
     """
     return None
 
+
 def randomBoolean():
     """Provide a random value of the Avro `boolean` type.
     """
     return random.choice([True, False])
 
+
 def randomInt():
     """Provide a random value of the Avro (32 bit, signed) `int` type.
     """
     return int(numpy.random.randint(-2**31, 2**31 - 1, dtype=numpy.int32))
 
+
 def randomLong():
     """Provide a random value of the Avro (64 bit, signed) `long` type.
     """
     return int(numpy.random.randint(-2**63, 2**63 - 1, dtype=numpy.int64))
 
+
 def randomFloat():
     """Provide a random value of the Avro (32) bit `float` type.
     """
     return float(numpy.float32(numpy.random.random()))
 
+
 def randomDouble():
     """Provide a random value of the Avro `double` type.
     """
     return float(numpy.float64(numpy.random.random()))
 
+
 def randomString():
     """Provide a random value of the Avro `string` type.
 
@@ -65,13 +72,15 @@ def randomString():
     return ''.join(random.choice(string.ascii_letters)
                    for _ in range(random.randint(0, 10)))
 
+
 def randomBytes(max_bytes=1000):
     """Provide a random value of the Avro `bytes` type.
 
     Up to `max_bytes` bytes are returned.
     """
     return numpy.random.bytes(random.randint(0, max_bytes))
 
+
 randomizerFunctionsByType = {
     'null': randomNull,
     'boolean': randomBoolean,
@@ -83,6 +92,7 @@ def randomBytes(max_bytes=1000):
     'bytes': randomBytes
 }
 
+
 def simulate_alert(schema, keepNull=None, arrayCount=None):
     """Parse the schema and generate a compliant alert with random contents.
 
@@ -94,7 +104,8 @@ def simulate_alert(schema, keepNull=None, arrayCount=None):
     keepNull : {`list` of `str`, `None`}
         Schema keys for which to output null values.
     arrayCount : {`dict`, `None`}
-        Number of array items to randomly generate for each provided schema key.
+        Number of array items to randomly generate for each provided schema
+        key.
 
     Returns
     -------

setup.cfg

@@ -35,3 +35,11 @@ console_scripts =
     validateAvroRoundTrip.py = lsst.alert.packet.bin.validateAvroRoundTrip:main
     simulateAlerts.py = lsst.alert.packet.bin.simulateAlerts:main
     syncLatestSchemaToRegistry.py = lsst.alert.packet.bin.syncLatestSchemaToRegistry:main
+
+[flake8]
+max-line-length = 110
+max-doc-length = 79
+ignore = E133, E226, E228, N802, N803, N806, N812, N813, N815, N816, W503
+exclude =
+    __init__.py
+    doc/conf.py