more tests and update scripts

pioreactor/background_jobs/od_reading.py

@@ -1018,7 +1018,9 @@ def _determine_best_ir_led_intensity(
         ) / 50  # divide by N since the culture is unlikely to Nx.
 
         ir_intensity_max = 80.0
-        return min(ir_intensity_max, ir_intensity_argmax_ANGLE_can_be, ir_intensity_argmax_REF_can_be)
+        return round(
+            min(ir_intensity_max, ir_intensity_argmax_ANGLE_can_be, ir_intensity_argmax_REF_can_be), 2
+        )
 
     def _prepare_post_callbacks(self) -> list[Callable]:
         callbacks: list[Callable] = []

pioreactor/tests/test_execute_experiment_profile.py

@@ -239,9 +239,28 @@ def test_execute_experiment_start_automation_succeeds(
     execute_experiment_profile("profile.yaml", experiment)
 
 
-@pytest.mark.xfail(reason="need to write a good test for this")
-def test_label_fires_a_relabel_to_leader_endpoint():
-    assert False
+@patch("pioreactor.actions.leader.experiment_profile._load_experiment_profile")
+def test_label_fires_a_relabel_to_leader_endpoint(mock__load_experiment_profile):
+    experiment = "_testing_experiment"
+
+    profile = Profile(
+        experiment_profile_name="test_profile",
+        plugins=[],
+        pioreactors={
+            "unit1": PioreactorSpecificBlock(label="label1"),
+            "unit2": PioreactorSpecificBlock(label="label2"),
+        },
+        metadata=Metadata(author="test_author"),
+    )
+
+    mock__load_experiment_profile.return_value = profile
+
+    with capture_requests() as bucket:
+        execute_experiment_profile("profile.yaml", experiment)
+
+    assert bucket[0].path == "/api/experiments/_testing_experiment/unit_labels"
+    assert bucket[0].json == {"label": "label1", "unit": "unit1"}
+    assert bucket[1].json == {"label": "label2", "unit": "unit2"}
 
 
 @patch("pioreactor.actions.leader.experiment_profile._load_experiment_profile")
@@ -416,7 +435,6 @@ def test_wrong_syntax_in_if_statement(mock__load_experiment_profile) -> None:
 
 @patch("pioreactor.actions.leader.experiment_profile._load_experiment_profile")
 def test_repeat_block(mock__load_experiment_profile) -> None:
-    experiment = "_testing_experiment"
     repeat_num = 6
     repeat_every_hours = 0.001
     start = Start(hours_elapsed=0)
@@ -443,20 +461,15 @@ def test_repeat_block(mock__load_experiment_profile) -> None:
 
     mock__load_experiment_profile.return_value = profile
 
-    actions = []
-
-    def collect_actions(msg):
-        actions.append(msg.payload.decode())
-
-    subscribe_and_callback(
-        collect_actions,
-        [f"pioreactor/unit1/{experiment}/jobbing/setting/set"],
-        allow_retained=False,
-    )
-
-    execute_experiment_profile("profile.yaml", experiment)
+    with capture_requests() as bucket:
+        execute_experiment_profile("profile.yaml", "_testing_experiment")
 
-    assert actions == ["1"] * repeat_num
+    r = [
+        b.json["settings"]["setting"]
+        for b in bucket
+        if b.path == "/api/workers/unit1/jobs/update/job_name/jobbing/experiments/_testing_experiment"
+    ]
+    assert r == ["1"] * repeat_num
 
 
 @patch("pioreactor.actions.leader.experiment_profile._load_experiment_profile")
@@ -565,31 +578,21 @@ def test_execute_experiment_profile_when_action_simple(mock__load_experiment_pro
 
     mock__load_experiment_profile.return_value = profile
 
-    actions = []
-
-    def collect_actions(msg):
-        actions.append(msg.topic)
-
-    subscribe_and_callback(
-        collect_actions,
-        [f"pioreactor/unit1/{experiment}/#"],
-        allow_retained=False,
-    )
-
     # Simulate OD value
     publish(
         f"pioreactor/unit1/{experiment}/od_reading/od1",
         encode(ODReading(od=2.5, angle="90", timestamp=current_utc_datetime(), channel="1")),
         retain=True,
     )
 
-    execute_experiment_profile("profile.yaml", experiment)
+    with capture_requests() as bucket:
+        execute_experiment_profile("profile.yaml", experiment)
 
-    assert actions == [
-        f"pioreactor/unit1/{experiment}/od_reading/od1",
-        f"pioreactor/unit1/{experiment}/run/stirring",
-        f"pioreactor/unit1/{experiment}/stirring/target_rpm/set",
-    ]
+    assert len(bucket) == 2
+    assert bucket[0].path == "/api/workers/unit1/jobs/run/job_name/stirring/experiments/_testing_experiment"
+    assert (
+        bucket[1].path == "/api/workers/unit1/jobs/update/job_name/stirring/experiments/_testing_experiment"
+    )
 
 
 @patch("pioreactor.actions.leader.experiment_profile._load_experiment_profile")
@@ -618,31 +621,21 @@ def test_execute_experiment_profile_when_action_with_if(mock__load_experiment_pr
 
     mock__load_experiment_profile.return_value = profile
 
-    actions = []
-
-    def collect_actions(msg):
-        actions.append(msg.topic)
-
-    subscribe_and_callback(
-        collect_actions,
-        [f"pioreactor/unit1/{experiment}/#"],
-        allow_retained=False,
-    )
-
     # Simulate OD value
     publish(
         f"pioreactor/unit1/{experiment}/od_reading/od1",
         encode(ODReading(od=2.5, angle="90", timestamp=current_utc_datetime(), channel="1")),
         retain=True,
     )
 
-    execute_experiment_profile("profile.yaml", experiment)
+    with capture_requests() as bucket:
+        execute_experiment_profile("profile.yaml", experiment)
 
-    assert actions == [
-        f"pioreactor/unit1/{experiment}/od_reading/od1",
-        f"pioreactor/unit1/{experiment}/run/stirring",
-        f"pioreactor/unit1/{experiment}/stirring/target_rpm/set",
-    ]
+    assert len(bucket) == 2
+    assert bucket[0].path == "/api/workers/unit1/jobs/run/job_name/stirring/experiments/_testing_experiment"
+    assert (
+        bucket[1].path == "/api/workers/unit1/jobs/update/job_name/stirring/experiments/_testing_experiment"
+    )
 
 
 @patch("pioreactor.actions.leader.experiment_profile._load_experiment_profile")

update_scripts/upcoming/lighttpd/50-pioreactorui.conf

@@ -0,0 +1,20 @@
+
+url.rewrite-once = (
+ "^(/static($|/.*))$" => "$1",
+ "^(/api/.*)$" => "/main.fcgi$1",
+ "^(/unit_api/.*)$" => "/main.fcgi$1",
+)
+alias.url += (
+  "/static/" => "/var/www/pioreactorui/static/"
+)
+server.error-handler-404 = "/index.html"
+
+
+fastcgi.server += (".fcgi" =>
+    ((
+        "socket" => "/tmp/main-fcgi.sock",
+        "bin-path" => "/var/www/pioreactorui/main.fcgi",
+        "check-local" => "disable",
+        "max-procs" => 1,
+    ))
+)

update_scripts/upcoming/lighttpd/52-api-only.conf

@@ -0,0 +1,4 @@
+
+$HTTP["url"] !~ "^(/unit_api|/main.fcgi/unit_api)" {
+    url.access-deny = ( "" )
+}

update_scripts/upcoming/lighttpd/lighttpd.conf

@@ -0,0 +1,49 @@
+server.modules = (
+    "mod_indexfile",
+    "mod_access",
+    "mod_alias",
+    "mod_redirect",
+    "mod_staticfile",
+)
+
+server.document-root        = "/var/www/pioreactorui"
+server.upload-dirs          = ( "/var/cache/lighttpd/uploads" )
+server.errorlog             = "/var/log/lighttpd/error.log"
+server.pid-file             = "/run/lighttpd.pid"
+server.username             = "www-data"
+server.groupname            = "www-data"
+server.port                 = 80
+
+# features
+#https://redmine.lighttpd.net/projects/lighttpd/wiki/Server_feature-flagsDetails
+server.feature-flags       += ("server.h2proto" => "enable")
+server.feature-flags       += ("server.h2c"     => "enable")
+server.feature-flags       += ("server.graceful-shutdown-timeout" => 5)
+#server.feature-flags       += ("server.graceful-restart-bg" => "enable")
+
+# strict parsing and normalization of URL for consistency and security
+# https://redmine.lighttpd.net/projects/lighttpd/wiki/Server_http-parseoptsDetails
+# (might need to explicitly set "url-path-2f-decode" = "disable"
+#  if a specific application is encoding URLs inside url-path)
+server.http-parseopts = (
+  "header-strict"           => "enable",# default
+  "host-strict"             => "enable",# default
+  "host-normalize"          => "enable",# default
+  "url-normalize-unreserved"=> "enable",# recommended highly
+  "url-normalize-required"  => "enable",# recommended
+  "url-ctrls-reject"        => "enable",# recommended
+  "url-path-2f-decode"      => "enable",# recommended highly (unless breaks app)
+ #"url-path-2f-reject"      => "enable",
+  "url-path-dotseg-remove"  => "enable",# recommended highly (unless breaks app)
+ #"url-path-dotseg-reject"  => "enable",
+ #"url-query-20-plus"       => "enable",# consistency in query string
+)
+
+index-file.names            = ( "index.php", "index.html" )
+url.access-deny             = ( "~", ".inc" )
+static-file.exclude-extensions = ( ".php", ".pl", ".fcgi" )
+
+# default listening port for IPv6 falls back to the IPv4 port
+include_shell "/usr/share/lighttpd/use-ipv6.pl " + server.port
+include_shell "/usr/share/lighttpd/create-mime.conf.pl"
+include "/etc/lighttpd/conf-enabled/*.conf"

update_scripts/upcoming/scripts/create_diskcache.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -x
+set -e
+
+export LC_ALL=C
+
+DIR=/tmp/pioreactorui_cache
+
+mkdir -p $DIR
+
+touch $DIR/cache.db
+touch $DIR/cache.db-shm
+touch $DIR/cache.db-wal
+
+touch $DIR/huey.db
+touch $DIR/huey.db-shm
+touch $DIR/huey.db-wal
+
+chmod -R 770 $DIR/
+chown -R pioreactor:www-data $DIR/
+chmod g+s $DIR

update_scripts/upcoming/scripts/update_ui.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+set -x
+set -e
+
+export LC_ALL=C
+
+SRC_TAR=$1
+TEMP_DIR=$(mktemp -d -t "pioreactorui_XXXX")
+UI_FOLDER=/var/www/pioreactorui
+
+function finish {
+    # cleanup
+    rm -rf "$TEMP_DIR" || true
+    sudo systemctl restart lighttpd.service
+    sudo systemctl restart huey.service
+}
+trap finish EXIT
+
+
+# unpack source provided
+tar -xzf "$SRC_TAR" -C $TEMP_DIR
+WORK_DIR=$(find "$TEMP_DIR" -mindepth 1 -maxdepth 1 -type d) # get the directory inside the archive, name is not predictable.
+
+echo $WORK_DIR
+# Verify that WORK_DIR is valid
+if [[ -z "$WORK_DIR" ]]; then
+    echo "Failed to find the working directory inside TEMP_DIR"
+    exit 1
+fi
+
+# copy data over
+# use rsync because we want to merge custom yamls the user has, we any updates to our own yamls.
+rsync -ap --ignore-existing $UI_FOLDER/contrib/ $WORK_DIR/contrib/ 2>/dev/null || :
+
+if [ -f "$UI_FOLDER/.env" ]; then
+    echo "Copying .env file"
+    cp -p $UI_FOLDER/.env $WORK_DIR
+else
+    echo ".env file does not exist in $UI_FOLDER"
+fi
+
+# swap folders
+rm -rf $UI_FOLDER
+mkdir $UI_FOLDER
+cp -rp $WORK_DIR/. $UI_FOLDER
+chgrp -R www-data $UI_FOLDER
+
+ENV=$UI_FOLDER/.env
+if [ -f "$ENV" ]; then
+    echo "$ENV exists."
+else
+    mv $UI_FOLDER/.env.example $ENV
+fi

update_scripts/upcoming/services/create_diskcache.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Prepare cache for UI
+After=network.target firstboot.service
+Before=lighttpd.service huey.service
+
+[Service]
+User=pioreactor
+ExecStart=/bin/bash /usr/local/bin/create_diskcache.sh
+ExecStartPost=pio log -m "%n successful" -n systemd --local-only
+Type=oneshot
+StandardOutput=null
+StandardError=null
+
+[Install]
+WantedBy=multi-user.target

update_scripts/upcoming/services/huey.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Huey service for PioreactorUI
+After=network.target firstboot.service
+Before=lighttpd.service
+
+[Service]
+User=pioreactor
+WorkingDirectory=/var/www/pioreactorui
+ExecStart=huey_consumer tasks.huey -n -b 1.0 -w 2 -f -C
+ExecStartPost=pio log -m "%n successful" -n systemd --local-only
+StandardOutput=null
+StandardError=null
+
+[Install]
+WantedBy=multi-user.target

update_scripts/upcoming/services/lighttpd.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Lighttpd Daemon
+After=network-online.target
+
+[Service]
+Type=simple
+PIDFile=/run/lighttpd.pid
+ExecStartPre=/usr/sbin/lighttpd -tt -f /etc/lighttpd/lighttpd.conf
+ExecStart=/usr/sbin/lighttpd -D -f /etc/lighttpd/lighttpd.conf
+ExecReload=/bin/kill -USR1 $MAINPID
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target