more

CHANGELOG.md

@@ -4,7 +4,7 @@
  - Fix for UI labels when trying to remove multiple labels from Pioreactors.
  - Added groupings on the Experiment dropdown to organize "Active" and "Inactive" experiments. An active experiment has >= 1 Pioreactor assigned to it.
  - UI now supports changing the MQTT broker. This configuration lives in the config.ini, under `[mqtt]`.
- -
+ - New log topic that partitions by the level. This should make subscribers to the log topic slimmer (like the UI, who would have to accept and filter _all_ messages). Should result in a performance increase.
 
 
 ### 24.5.22

config.dev.ini

@@ -1,10 +1,10 @@
 [PWM]
-# map the PWM channels to externals.
-# hardware PWM are available on channels 2 & 4.
+# map the externals to the PWM
+# hardware PWM are available on channels 1 & 3.
 1=stirring
-2=media
-3=alt_media
-4=waste
+2=waste
+3=media
+4=alt_media
 5=heating
 
 

pioreactor/actions/led_intensity.py

@@ -25,11 +25,12 @@
 from pioreactor.whoami import is_testing_env
 
 ALL_LED_CHANNELS: list[LedChannel] = ["A", "B", "C", "D"]
+LEDsToIntensityMapping = dict[LedChannel, LedIntensityValue]
 
 
 @contextmanager
 def change_leds_intensities_temporarily(
-    desired_state: dict[LedChannel, LedIntensityValue],
+    desired_state: LEDsToIntensityMapping,
     **kwargs: Any,
 ) -> Iterator[None]:
     """
@@ -69,32 +70,32 @@ def is_led_channel_locked(channel: LedChannel) -> bool:
 
 
 def _update_current_state(
-    state: dict[LedChannel, LedIntensityValue],
-) -> tuple[structs.LEDsIntensity, structs.LEDsIntensity]:
+    state: LEDsToIntensityMapping,
+) -> tuple[LEDsToIntensityMapping, LEDsToIntensityMapping]:
     """
     TODO: Eventually I should try to modify the UI to not even need this `state` variable,
     """
 
     with local_intermittent_storage("leds") as led_cache:
         # rehydrate old cache
-        old_state = structs.LEDsIntensity(
-            **{str(channel): led_cache.get(str(channel), 0.0) for channel in ALL_LED_CHANNELS}
-        )
+        old_state: LEDsToIntensityMapping = {
+            channel: led_cache.get(str(channel), 0.0) for channel in ALL_LED_CHANNELS
+        }
 
         # update cache
         with led_cache.transact():
             for channel, intensity in state.items():
                 led_cache[channel] = intensity
 
-        new_state = structs.LEDsIntensity(
-            **{str(channel): led_cache.get(str(channel), 0.0) for channel in ALL_LED_CHANNELS}
-        )
+        new_state: LEDsToIntensityMapping = {
+            channel: led_cache.get(str(channel), 0.0) for channel in ALL_LED_CHANNELS
+        }
 
         return new_state, old_state
 
 
 def led_intensity(
-    desired_state: dict[LedChannel, LedIntensityValue],
+    desired_state: LEDsToIntensityMapping,
     unit: str | None = None,
     experiment: str | None = None,
     verbose: bool = True,
@@ -244,7 +245,7 @@ def led_intensity(
     type=str,
     help="whom is calling this function (for logging purposes)",
 )
-@click.option("--no-log", is_flag=True, help="Add to log")
+@click.option("--no-log", is_flag=True, help="skip logging")
 def click_led_intensity(
     a: LedIntensityValue | None = None,
     b: LedIntensityValue | None = None,
@@ -259,7 +260,7 @@ def click_led_intensity(
     unit = get_unit_name()
     experiment = get_assigned_experiment_name(unit)
 
-    state: dict[LedChannel, LedIntensityValue] = {}
+    state: LEDsToIntensityMapping = {}
     if a is not None:
         state["A"] = a
     if b is not None:

pioreactor/actions/pump.py

@@ -306,7 +306,7 @@ def _pump_action(
                     ml = DEFAULT_PWM_CALIBRATION.duration_to_ml(duration)  # naive
                 logger.info(_to_human_readable_action(None, duration, pump_type))
             elif continuously:
-                duration = 10.0
+                duration = 2.5
                 try:
                     ml = pump.duration_to_ml(duration)  # can be wrong if calibration is not defined
                 except exc.CalibrationError:

pioreactor/actions/self_test.py

@@ -62,7 +62,7 @@ def test_REF_is_in_correct_position(client: Client, logger: CustomLogger, unit:
     # this _also_ uses stirring to increase the variance in the non-REF.
     # The idea is to trigger stirring on and off and the REF should not see a change in signal / variance, but the other PD should.
 
-    assert is_HAT_present()
+    assert is_HAT_present(), "Hat is not detected."
 
     reference_channel = cast(PdChannel, config["od_config.photodiode_channel_reverse"][REF_keyword])
     signal_channel = "2" if reference_channel == "1" else "1"
@@ -119,7 +119,7 @@ def test_all_positive_correlations_between_pds_and_leds(
     """
     from pprint import pformat
 
-    assert is_HAT_present()
+    assert is_HAT_present(), "HAT is not detected."
     # better to err on the side of MORE samples than less - it's only a few extra seconds...
     # we randomize to reduce effects of temperature
     # upper bound shouldn't be too high, as it could saturate the ADC, and lower bound shouldn't be too low, else we don't detect anything.
@@ -233,7 +233,7 @@ def test_all_positive_correlations_between_pds_and_leds(
 
 def test_ambient_light_interference(client: Client, logger: CustomLogger, unit: str, experiment: str) -> None:
     # test ambient light IR interference. With all LEDs off, and the Pioreactor not in a sunny room, we should see near 0 light.
-    assert is_HAT_present()
+    assert is_HAT_present(), "HAT is not detected."
     adc_reader = ADCReader(
         channels=ALL_PD_CHANNELS,
         dynamic_gain=False,
@@ -298,7 +298,7 @@ def test_REF_is_lower_than_0_dot_256_volts(
 def test_PD_is_near_0_volts_for_blank(
     client: Client, logger: CustomLogger, unit: str, experiment: str
 ) -> None:
-    assert is_HAT_present()
+    assert is_HAT_present(), "HAT is not detected."
     reference_channel = cast(PdChannel, config["od_config.photodiode_channel_reverse"][REF_keyword])
 
     if reference_channel == "1":
@@ -308,7 +308,7 @@ def test_PD_is_near_0_volts_for_blank(
 
     angle = config.get("od_config.photodiode_channel", signal_channel, fallback=None)
 
-    assert angle in ["90", "45", "135"]
+    assert angle in ["90", "45", "135"], f"Angle {angle} not valid for this test."
 
     signals = []
 
@@ -340,7 +340,7 @@ def test_detect_heating_pcb(client: Client, logger: CustomLogger, unit: str, exp
 def test_positive_correlation_between_temperature_and_heating(
     client, logger: CustomLogger, unit: str, experiment: str
 ) -> None:
-    assert is_heating_pcb_present()
+    assert is_heating_pcb_present(), "Heater PCB is not connected, or i2c is not working."
 
     with TemperatureController(unit, experiment, "only_record_temperature") as tc:
         measured_pcb_temps = []
@@ -360,16 +360,16 @@ def test_positive_correlation_between_temperature_and_heating(
 
 
 def test_aux_power_is_not_too_high(client: Client, logger: CustomLogger, unit: str, experiment: str) -> None:
-    assert is_HAT_present()
-    assert voltage_in_aux() <= 18.0
+    assert is_HAT_present(), "HAT was not detected."
+    assert voltage_in_aux() <= 18.0, f"Voltage measured {voltage_in_aux()} > 18.0V"
 
 
 def test_positive_correlation_between_rpm_and_stirring(
     client, logger: CustomLogger, unit: str, experiment: str
 ) -> None:
-    assert is_HAT_present()
-    assert is_heating_pcb_present()
-    assert voltage_in_aux() <= 18.0
+    assert is_HAT_present(), "HAT was not detected."
+    assert is_heating_pcb_present(), "Heating PCB was not detected."
+    assert voltage_in_aux() <= 18.0, f"Voltage measured {voltage_in_aux()} > 18.0V"
 
     with local_persistant_storage("stirring_calibration") as cache:
         if "linear_v1" in cache:
@@ -408,7 +408,7 @@ def test_positive_correlation_between_rpm_and_stirring(
         measured_correlation = round(correlation(dcs, measured_rpms), 2)
         logger.debug(f"Correlation between stirring RPM and duty cycle: {measured_correlation}")
         logger.debug(f"{dcs=}, {measured_rpms=}")
-        assert measured_correlation > 0.9, (dcs, measured_rpms)
+        assert measured_correlation > 0.9, f"RPM correlation not high enough: {(dcs, measured_rpms)}"
 
 
 class BatchTestRunner:
@@ -437,7 +437,7 @@ def _run(self, client, logger: CustomLogger, unit: str, testing_experiment: str)
                 res = True
             except Exception as e:
                 logger.debug(e, exc_info=True)
-                logger.warning(f" in {test_name.replace('_', ' ')}: {e}")
+                logger.warning(f"{test_name.replace('_', ' ')}: {e}")
 
             logger.debug(f"{test_name}: {'✅' if res else '❌'}")
 

pioreactor/background_jobs/growth_rate_calculating.py

@@ -77,9 +77,8 @@ class GrowthRateCalculator(BackgroundJob):
             "datatype": "GrowthRate",
             "settable": False,
             "unit": "h⁻¹",
-            "persist": True,  # TODO: why persist?
         },
-        "od_filtered": {"datatype": "ODFiltered", "settable": False, "persist": True},
+        "od_filtered": {"datatype": "ODFiltered", "settable": False},
         "kalman_filter_outputs": {
             "datatype": "KalmanFilterOutput",
             "settable": False,

pioreactor/background_jobs/leader/mqtt_to_db_streaming.py

@@ -401,7 +401,7 @@ def add_default_source_to_sinks() -> list[TopicToParserToTable]:
                 parse_alt_media_fraction,
                 "alt_media_fractions",
             ),
-            TopicToParserToTable("pioreactor/+/+/logs/+", parse_logs, "logs"),
+            TopicToParserToTable("pioreactor/+/+/logs/#", parse_logs, "logs"),
             TopicToParserToTable(
                 "pioreactor/+/+/dosing_automation/dosing_automation_settings",
                 parse_automation_settings,

pioreactor/logging.py

@@ -119,7 +119,11 @@ def emit(self, record) -> None:
                 return
 
         mqtt_msg = self.client.publish(
-            self.topic, payload, qos=self.qos, retain=self.retain, **self.mqtt_kwargs
+            f"{self.topic}/{record.levelname.lower()}",
+            payload,
+            qos=self.qos,
+            retain=self.retain,
+            **self.mqtt_kwargs,
         )
         # if Python exits too quickly, the last msg might never make it to the broker.
         mqtt_msg.wait_for_publish(timeout=2)

pioreactor/structs.py

@@ -122,13 +122,6 @@ class LEDChangeEvent(Struct):
     timestamp: t.Annotated[datetime, Meta(tz=True)]
 
 
-class LEDsIntensity(Struct):
-    A: pt.LedIntensityValue = 0.0
-    B: pt.LedIntensityValue = 0.0
-    C: pt.LedIntensityValue = 0.0
-    D: pt.LedIntensityValue = 0.0
-
-
 class DosingEvent(Struct):
     """
     Output of a pump action

pioreactor/utils/math_helpers.py

@@ -38,9 +38,7 @@ def trimmed_mean(x: Sequence) -> float:
     return mean(x)
 
 
-def simple_linear_regression(
-    x: Sequence, y: Sequence
-) -> tuple[tuple[float, float], tuple[float, float]]:
+def simple_linear_regression(x: Sequence, y: Sequence) -> tuple[tuple[float, float], tuple[float, float]]:
     from statistics import linear_regression
 
     n = len(x)
@@ -71,7 +69,7 @@ def simple_linear_regression_with_forced_nil_intercept(
     from statistics import linear_regression
 
     n = len(x)
-    assert n > 2, "not enough data points for linear regression"
+    assert n >= 2, "not enough data points for linear regression"
     assert n == len(y), "Array sizes are not equal."
 
     # Compute the regression using statistics.linear_regression