Refactor software env, update cnn examples.

.flake8

@@ -4,7 +4,7 @@
 max-line-length = 120
 
 # codes of errors to ignore
-ignore = E128, E306, E402, E722, E731, W504
+ignore = E128, E306, E402, E702, E722, E731, W504
 
 # enforce double quotes
 inline-quotes = double

.github/workflows/lint_and_test.yml

@@ -0,0 +1,48 @@
+name: Lint and test
+
+on:
+  workflow_dispatch:
+  push:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout ⬇️
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+
+      - name: Setup python 🐍
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies ☕️
+        run: |
+          pip install -U pip setuptools wheel
+          pip install -r sandboxes/dev.txt
+
+      - name: Lint 🔍
+        run: flake8 mlprof
+
+  typecheck:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout ⬇️
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+
+      - name: Setup python 🐍
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies ☕️
+        run: |
+          pip install -U pip setuptools wheel
+          pip install -r sandboxes/dev.txt
+
+      - name: Typecheck 🥊
+        run: mypy mlprof

.gitignore

@@ -14,6 +14,7 @@
 *.hdf5
 *.out
 *.parquet
+*.vscode
 .coverage
 coverage*.xml
 __pycache__
@@ -26,3 +27,4 @@ software
 data
 .data
 .law
+.python-version

examples/cnn/create.py

@@ -0,0 +1,106 @@
+# coding: utf-8
+
+"""
+Test script to create a simple CNN model.
+Best to be executed in a CMSSW environment with TensorFlow and cmsml installed or a dedicated virtual environment.
+
+Signature: f32(1024 * n_channels) -> f32(8)
+
+1024 could correspond to a pixelated 32x32 "image" of an eta-phi plane.
+"""
+
+import os
+import subprocess
+
+import cmsml  # type: ignore[import-untyped]
+
+
+def create_model(
+    model_dir: str,
+    postfix: str = r"i{n_in}c{n_channels}v{n_convs}l{n_layers}u{n_units}",
+    n_in: int = 32,
+    n_channels: int = 1,
+    n_filters: int = 32,
+    n_convs: int = 3,
+    n_out: int = 8,
+    n_layers: int = 10,
+    n_units: int = 128,
+) -> None:
+    # get tensorflow
+    tf, _, tf_version = cmsml.tensorflow.import_tf()
+    print("creating simple model")
+    print(f"location  : {model_dir}")
+    print(f"TF version: {'.'.join(map(str, tf_version))}")
+
+    # set random seeds to get deterministic results for testing
+    tf.keras.utils.set_random_seed(1)
+
+    # define input layer
+    x = tf.keras.Input(shape=(n_channels * n_in**2,), dtype=tf.float32, name="input")
+
+    # reshape
+    a = tf.keras.layers.Reshape((n_in, n_in, n_channels))(x)
+
+    # convolutions and pooling
+    for _ in range(n_convs):
+        a = tf.keras.layers.Conv2D(n_filters, (3, 3), activation="elu")(a)
+        a = tf.keras.layers.MaxPooling2D((2, 2))(a)
+
+    # flatten
+    a = tf.keras.layers.Flatten()(a)
+
+    # model layers
+    for _ in range(n_layers):
+        a = tf.keras.layers.Dense(n_units, activation="elu")(a)
+
+    # output layer
+    y = tf.keras.layers.Dense(n_out, activation="softmax", name="output", dtype=tf.float32)(a)
+
+    # define the model
+    model = tf.keras.Model(inputs=[x], outputs=[y])
+
+    # test evaluation
+    print(model([tf.constant([list(range(n_channels * n_in**2))], dtype=tf.float32)]))
+
+    # save it as a frozen graph
+    _postfix = postfix.format(
+        n_in=n_in,
+        n_out=n_out,
+        n_channels=n_channels,
+        n_filters=n_filters,
+        n_convs=n_convs,
+        n_layers=n_layers,
+        n_units=n_units,
+    )
+    cmsml.tensorflow.save_graph(
+        os.path.join(model_dir, f"frozen_graph_{_postfix}.pb"),
+        model,
+        variables_to_constants=True,
+    )
+
+    # create a SavedModel
+    saved_model_dir = os.path.join(model_dir, f"saved_model_{_postfix}")
+    tf.saved_model.save(model, saved_model_dir)
+
+    # convert to onnx
+    cmd = [
+        "python",
+        "-m", "tf2onnx.convert",
+        "--saved-model", saved_model_dir,
+        "--output", os.path.join(model_dir, f"onnx_graph_{_postfix}.onnx"),
+    ]
+    subprocess.run(cmd, check=True)
+
+
+def main():
+    this_dir = os.path.dirname(os.path.abspath(__file__))
+
+    create_model(this_dir, n_in=32, n_channels=1, n_convs=1, n_layers=10)
+    create_model(this_dir, n_in=32, n_channels=1, n_convs=3, n_layers=10)
+    create_model(this_dir, n_in=64, n_channels=3, n_convs=4, n_layers=10)
+    create_model(this_dir, n_in=32, n_channels=1, n_convs=3, n_layers=20)
+    create_model(this_dir, n_in=64, n_channels=3, n_convs=4, n_layers=20)
+
+
+if __name__ == "__main__":
+    main()

examples/cnn/model_onnx_l1k1f4.yaml → examples/cnn/model_onnx_i32c1v1l10u128.yaml

@@ -1,11 +1,11 @@
 model:
   name: simple_cnn_onnx
-  label: ONNX 1x4x1
+  label: ONNX 32x1$\circledast$1$\rightarrow$10x128
   version: "1.0.0"
   inference_engine: onnx
-  file: ./onnx_graph_l1k1f4.onnx
+  file: ./onnx_graph_i32c1v1l10u128.onnx
   inputs:
     - name: input
-      shape: [28, 28, 1]
+      shape: [1024]
   outputs:
     - name: output

examples/cnn/model_onnx_l1k3f4.yaml → examples/cnn/model_onnx_i32c1v3l10u128.yaml

@@ -1,11 +1,11 @@
 model:
   name: simple_cnn_onnx
-  label: ONNX 1x4x3
+  label: ONNX 32x1$\circledast$3$\rightarrow$10x128
   version: "1.0.0"
   inference_engine: onnx
-  file: ./onnx_graph_l1k3f4.onnx
+  file: ./onnx_graph_i32c1v3l10u128.onnx
   inputs:
     - name: input
-      shape: [28,28,1]
+      shape: [1024]
   outputs:
     - name: output

examples/cnn/model_onnx_l5k1f4.yaml → examples/cnn/model_onnx_i32c1v3l20u128.yaml

@@ -1,11 +1,11 @@
 model:
   name: simple_cnn_onnx
-  label: ONNX 5x4x1
+  label: ONNX 32x1$\circledast$3$\rightarrow$20x128
   version: "1.0.0"
   inference_engine: onnx
-  file: ./onnx_graph_l5k1f4.onnx
+  file: ./onnx_graph_i32c1v3l20u128.onnx
   inputs:
     - name: input
-      shape: [28,28,1]
+      shape: [1024]
   outputs:
     - name: output