DM-24283: Add updateSchema tool

.github/workflows/build.yml

@@ -14,6 +14,9 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Install yaml needed for build
+      run: |
+        pip install pyyaml
     - name: Build and install wheel
       run: |
         pip install virtualenv

README.rst

@@ -34,24 +34,26 @@ Steps to update the alert schema (for example, when the APDB schema is updated).
 * ``setup -r .`` in this package's root.
 * Checkout the ticket branch for your schema changes.
 * Update the default ``schema_root`` kwarg in ``python/lsst/alert/packet/schemaRegistry.py:from_filesystem()`` to your new schema version number.
-* Add a new schema in ``python/lsst/alert/packet/schema/``. Subsequent instructions assume you are in that directory.
 
-   * Recursively copy the latest schema directory to the next number (e.g. ``cp -rp 4/ 5/`` to add schema version 5 based on version 4, or ``cp -rp 4/0 4/1`` for a minor version bump). Subsequent directions assume you are in this new directory.
+   * New schemas are built from the apdb. Any changes to a field should be done in apdb.yaml, and any changes to what is included/excluded should be made in updateSchema.py.
+   * To update the schema, you must have the path to the apdb.yaml file and have chosen a version number. If the directory for your version
+   * does not already exist, ``updateSchema.py`` will create it.
 
-      * Rename all of the ``lsst.vX*`` files to the new version.
-      * Update the ``"namespace": "lsst.v5_0",`` line at the top of each ``*.avsc`` file to the new version.
+      * run ``python updateSchema.py /path/to/LSST/code/sdm_schemas/yml/apdb.yaml Path/To/alert_packet/lsst/alert/packet/schema "6.0"`` All Generated files do not need to be altered.
+      * Navigate to the new schema. Copy in the previous ``lsst.vX_X.alert.avsc`` file and ``lsst.vX_X.diaNondetectionLimit.avsc``.
+      * Within the two copied files, update ``"namespace": "lsst.vX_X",`` line at the top of each ``*.avsc`` file to the new version.
       * Update the contents of those avro schema files to reflect the new schema.
       * Update the sample alert packet in ``sample_data``:
 
          * Update ``alert.json`` to reflect the new schema.
-         * Change the ``schema_root`` and ``get_by_version`` parameters in ``generate.py`` to your new verison number.
+         * Change the ``schema_root`` and ``get_by_version`` parameters in ``generate.py`` to your new version number.
          * Run ``python generate.py`` to produce a new ``fakeAlert.avro`` file with data filled in from the updated json file above and using the new schema files you made earlier.
 
    * Update the files ``*.avsc`` and ``*.json`` files in ``examples/`` to reflect the new schema.
    * Update the contents of ``latest.txt`` to your new schema version number.
 
 * Add all of your new and updated files to ``git``, commit them with a message that includes the new schema version number and why it was incremented, and push your new branch.
-* Test your changes with ap_assocation and ap_verify (these steps may be more involved if your ticket branch impacts multiple packets):
+* Test your changes with ap_association and ap_verify (these steps may be more involved if your ticket branch impacts multiple packets):
 
    * Clone a local copy of `ap_association <https://github.com/lsst/ap_association/>`_ and cd into that directory.
    * Setup ap_association with ``setup -kr .`` (``-k`` to "keep" your previously setup ``alert_packet``).

python/lsst/alert/packet/__init__.py

@@ -23,3 +23,4 @@
 from .schema import *
 from .schemaRegistry import *
 from .simulate import *
+from .updateSchema import *

python/lsst/alert/packet/schema/6/0/lsst.v6_0.alert.avsc

@@ -0,0 +1,23 @@
+{
+	"namespace": "lsst.v6_0",
+	"type": "record",
+	"name": "alert",
+	"doc": "sample avro alert schema v6.0",
+	"fields": [
+		{"name": "alertId", "type": "long", "doc": "unique alert identifer"},
+		{"name": "diaSource", "type": "lsst.v6_0.diaSource"},
+		{"name": "prvDiaSources", "type": ["null", {
+				"type": "array",
+				"items": "lsst.v6_0.diaSource"}], "default": null},
+		{"name": "prvDiaForcedSources", "type": ["null", {
+				"type": "array",
+				"items": "lsst.v6_0.diaForcedSource"}], "default": null},
+		{"name": "prvDiaNondetectionLimits", "type": ["null", {
+				"type": "array",
+				"items": "lsst.v6_0.diaNondetectionLimit"}], "default": null},
+		{"name": "diaObject", "type": ["null", "lsst.v6_0.diaObject"], "default": null},
+		{"name": "ssObject", "type": ["null", "lsst.v6_0.ssObject"], "default": null},
+		{"name": "cutoutDifference", "type": ["null", "bytes"], "default": null},
+		{"name": "cutoutTemplate", "type": ["null", "bytes"], "default": null}
+     ]
+}

python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaForcedSource.avsc

@@ -0,0 +1,77 @@
+{
+  "type": "record",
+  "namespace": "lsst.v6_0",
+  "name": "diaForcedSource",
+  "fields": [
+    {
+      "doc": "Unique id.",
+      "name": "diaForcedSourceId",
+      "type": "long"
+    },
+    {
+      "doc": "Id of the DiaObject that this DiaForcedSource was associated with.",
+      "name": "diaObjectId",
+      "type": "long"
+    },
+    {
+      "doc": "Id of the ccdVisit where this forcedSource was measured.",
+      "name": "ccdVisitId",
+      "type": "long"
+    },
+    {
+      "default": null,
+      "doc": "Point Source model flux.",
+      "name": "psfFlux",
+      "type": [
+        "null",
+        "float"
+      ]
+    },
+    {
+      "default": null,
+      "doc": "Uncertainty of psfFlux.",
+      "name": "psfFluxErr",
+      "type": [
+        "null",
+        "float"
+      ]
+    },
+    {
+      "default": null,
+      "doc": "x position at which psfFlux has been measured.",
+      "name": "x",
+      "type": [
+        "null",
+        "float"
+      ]
+    },
+    {
+      "default": null,
+      "doc": "y position at which psfFlux has been measured.",
+      "name": "y",
+      "type": [
+        "null",
+        "float"
+      ]
+    },
+    {
+      "doc": "Flags, bitwise OR tbd",
+      "name": "flags",
+      "type": "long"
+    },
+    {
+      "doc": "Effective mid-visit time for this diaForcedSource, expressed as Modified Julian Date, International Atomic Time.",
+      "name": "midpointMjdTai",
+      "type": "double"
+    },
+    {
+      "default": null,
+      "doc": "Filter band this source was observed with.",
+      "name": "band",
+      "type": [
+        "null",
+        "string"
+      ]
+    }
+  ]
+}

python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaNondetectionLimit.avsc

@@ -0,0 +1,11 @@
+{
+	"namespace": "lsst.v6_0",
+	"name": "diaNondetectionLimit",
+	"type": "record",
+	"fields": [
+		{"name": "ccdVisitId",  "type": "long"},
+		{"name": "midpointMjdTai", "type": "double"},
+		{"name": "band", "type": "string"},
+		{"name": "diaNoise", "type": "float"}
+	]
+}