Uart serial readout to Influx time series database (#13)

* uart * dashboard * fix imports * influx * influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * default time range * uart influx * default time range * uart influx * ctrl layout * influxdb conf * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * dataselect * uart influx * systemd environmentFile * systemd environmentFile * uart batch size * dataselect defaults * dashboard * docs * force local clock * uart influx * uart write timestamp * uart influx * uart influx * uart influx * influxdb * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * dashboard * uart influx * uart influx * influxdb * uart influx * uart influxdb and dashboard * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * uart influx * dashboard UTC date time * uart influx * uart influx * dashboard * dashboard * uart service * uart service * uart influx * influx api * docs * multiprocessing
Multi-EAR · Dec 16, 2021 · d78c9c5 · d78c9c5
1 parent 2f74395
commit d78c9c5
Show file tree

Hide file tree

Showing 25 changed files with 1,418 additions and 942 deletions.
diff --git a/etc/influxdb/multi-ear.conf b/etc/influxdb/multi-ear.conf
@@ -8,13 +8,24 @@ bind-address = "127.0.0.1:8088"
 [data]
   dir = "/var/lib/influxdb/data"
   wal-dir = "/var/lib/influxdb/wal"
+  wal-fsync-delay = "100ms"
+  index-version = "inmem"
+  trace-logging-enabled = false
   query-log-enabled = false
-  cache-max-memory-size = "250m"
+  # cache-max-memory-size = "500m"
+  # cache-snapshot-write-cold-duration = "1m"
+  # compact-full-write-cold-duration = "30m"
   max-series-per-database = 0
+  max-values-per-tag = 0
 
 [retention]
   enabled = true
-  # check-interval = "3h"
+  # check-interval = "12h"
+
+[shard-precreation]
+  enabled = true
+  # check-interval = "10m"
+  # advance-period = "30m"
 
 [monitor]
   store-enabled = false
@@ -26,14 +37,22 @@ bind-address = "127.0.0.1:8088"
   bind-address = "127.0.0.1:8086"
   auth-enabled = false
   realm = "InfluxDB"
-  log-enabled = true
+  log-enabled = false
+  suppress-write-log = true
   write-tracing = false
   pprof-enabled = false
   pprof-auth-enabled = false
   debug-pprof-enabled = false
   ping-auth-enabled = false
   https-enabled = false
-  https-certificate = "/etc/ssl/influxdb.pem" 
-  unix-socket-enabled = true 
+  unix-socket-enabled = false 
   unix-socket-permissions = "0777"
   bind-socket = "/var/lib/influxdb/influxdb.sock"
+
+[logging]
+  format = "auto"
+  level = "info"
+  suppress-logo = true
+
+[subscriber]
+  enabled = false
diff --git a/etc/rsyslog.d/multi-ear.conf b/etc/rsyslog.d/multi-ear.conf
@@ -1,9 +1,9 @@
 if $programname == 'multi-ear-ctrl' then /var/log/multi-ear/ctrl.log
 & stop
-if $programname == 'multi-ear-data' then /var/log/multi-ear/data.log
-& stop
 if $programname == 'multi-ear-lora' then /var/log/multi-ear/lora.log
 & stop
+if $programname == 'multi-ear-sync' then /var/log/multi-ear/sync.log
+& stop
 if $programname == 'multi-ear-uart' then /var/log/multi-ear/uart.log
 & stop
 if $programname == 'multi-ear-wifi' then /var/log/multi-ear/wifi.log

diff --git a/etc/systemd/system/multi-ear-ctrl.service b/etc/systemd/system/multi-ear-ctrl.service
@@ -9,7 +9,10 @@ Group=tud
 WorkingDirectory=/home/tud/
 Environment="VIRTUAL_ENV=/home/tud/.py37"
 Environment="PATH=$VIRTUAL_ENV/bin:$PATH"
-ExecStart=/home/tud/.py37/bin/uwsgi --ini /home/tud/.py37/multi-ear-services/uwsgi.ini
+EnvironmentFile=/home/tud/.multi_ear.env
+ExecStart=
+ExecStart=/home/tud/.py37/bin/uwsgi \
+  --ini /home/tud/.py37/multi-ear-services/uwsgi.ini
 StandardOutput=syslog
 StandardError=syslog
 SyslogIdentifier=multi-ear-ctrl

diff --git a/etc/systemd/system/multi-ear-uart.service b/etc/systemd/system/multi-ear-uart.service
@@ -1,6 +1,7 @@
 [Unit]
-Description=Multi-EAR sensorboard serial readout via UART to a local InfluxDB time series database
-After=multi-user.target
+Description=Multi-EAR sensorboard serial readout to a local Influx time series database
+After=influxd.service
+# After=multi-user.target
 
 [Service]
 Type=simple
@@ -9,8 +10,11 @@ Group=tud
 WorkingDirectory=/home/tud/
 Environment="VIRTUAL_ENV=/home/tud/.py37"
 Environment="PATH=$VIRTUAL_ENV/bin:$PATH"
-# ExecStart=/home/tud/.py37/bin/python3 /home/tud/.py37/lib/python3.7/site-packages/multi_ear_services/uart/uart.py
-ExecStart=/home/tud/.py37/bin/multi-ear-uart
+EnvironmentFile=/home/tud/.multi_ear.env
+ExecStart=
+ExecStart=/home/tud/.py37/bin/multi-ear-uart \
+  --ini /home/tud/.py37/multi-ear-services/config.ini \
+  --journald
 StandardOutput=syslog
 StandardError=syslog
 SyslogIdentifier=multi-ear-uart

diff --git a/etc/systemd/system/multi-ear-wifi.timer b/etc/systemd/system/multi-ear-wifi.timer
diff --git a/multi-ear-services.sh b/multi-ear-services.sh
@@ -237,6 +237,20 @@ function do_systemd_service_configtest
 }
 
 
+function do_systemd_env
+{
+    cat > $HOME/.multi_ear.env << EOF
+MULTI_EAR_ID=$MULTI_EAR_ID
+MULTI_EAR_UUID=$MULTI_EAR_UUID
+MULTI_EAR_WIFI_SECRET=$MULTI_EAR_WIFI_SECRET
+INFLUX_USERNAME=$INFLUX_USERNAME
+INFLUX_PASSWORD=$INFLUX_PASSWORD
+GRAFANA_USERNAME=$GRAFANA_USERNAME
+GRAFANA_PASSWORD=$GRAFANA_PASSWORD
+EOF
+}
+
+
 #
 # Bashrc helpers
 #
@@ -495,46 +509,64 @@ function do_configure_hostapd
 
 function influx_e
 {
-    verbose_msg "> $1" 1
-    influx -execute "$1" >> $LOG_FILE 2>&1
+    if [ $# -eq 1 ]; then
+        verbose_msg "> $1" 1
+        influx -execute "$1" >> $LOG_FILE 2>&1
+    else
+        verbose_msg "> $1 @ $2" 1
+        influx -execute "$1" -database "$2" >> $LOG_FILE 2>&1
+    fi
 }
 
 
 function do_configure_influxdb
 {
     verbose_msg ".. configure influxdb"
+
     # enable
     do_systemd_service_enable "influxdb.service"
-    # link default settings
-    verbose_msg "> enable default configuration" 1
-    sudo ln -sf /etc/influxdb/default.conf /etc/influxdb/influxdb.conf >> $LOG_FILE 2>&1
+
+    ## link default settings
+    # verbose_msg "> enable default configuration" 1
+    # sudo ln -sf /etc/influxdb/default.conf /etc/influxdb/influxdb.conf >> $LOG_FILE 2>&1
+
+    # enforce multi-ear settings (requires login from now on!)
+    verbose_msg "> enable multi-ear configuration" 1
+    sudo ln -sf /etc/influxdb/multi-ear.conf /etc/influxdb/influxdb.conf >> $LOG_FILE 2>&1
+
     # restart with default settings
     do_systemd_service_restart "influxdb.service"
+
     # logging and output
     sudo mkdir -p /var/log/influxdb /var/lib/influxdb
     sudo chown -R influxdb:influxdb /var/log/influxdb /var/lib/influxdb
     sudo chmod 755 /var/log/influxdb /var/lib/influxdb
+
     #
-    # influx docs: https://docs.influxdata.com/influxdb/v1.8/
+    # influx query docs: https://docs.influxdata.com/influxdb/v1.8/
     #
+
+    # remove default internal database for logging
     influx_e "DROP DATABASE '_internal'"
-    # retention policies
-    local rp_m="onemonth" rp_m_specs="DURATION 30d REPLICATION 1 SHARD DURATION 5d"
-    local rp_y="oneyear"  rp_y_specs="DURATION 366d REPLICATION 1 SHARD DURATION 7d"
+
+    # create retention policies
+    # local rp_1m="one_month" rp_1m_specs="DURATION 30d REPLICATION 1 SHARD DURATION 6h DEFAULT"
+    local rp_2m="two_months" rp_2m_specs="DURATION 60d REPLICATION 1 SHARD DURATION 1d DEFAULT"
+
     # create database telegraf?
-    if ! influx -execute "show databases" | grep -q "telegraf";
+    if ! influx -execute "SHOW DATABASES" | grep -q "telegraf";
     then
         influx_e "CREATE DATABASE telegraf"
     fi
-    influx_e "USE DATABASE 'telegraf'"
-    influx_e "CREATE RETENTION POLICY $rp_m ON telegraf $rp_m_specs"
+    influx_e "CREATE RETENTION POLICY $rp_2m ON telegraf $rp_2m_specs"
+
     # create databases multi_ear?
-    if ! influx -execute "show databases" | grep -q "multi_ear";
+    if ! influx -execute "SHOW DATABASES" | grep -q "multi_ear";
     then
         influx_e "CREATE DATABASE multi_ear"
     fi
-    influx_e "USE DATABASE 'multi_ear'"
-    influx_e "CREATE RETENTION POLICY $rp_y ON multi_ear $rp_y_specs"
+    influx_e "CREATE RETENTION POLICY $rp_2m ON multi_ear $rp_2m_specs"
+
     # create full-privilege user
     if [ "$INFLUX_USERNAME" == "" ];
     then
@@ -552,21 +584,18 @@ function do_configure_influxdb
     fi
     influx_e "GRANT ALL PRIVILEGES ON multi_ear TO $INFLUX_USERNAME"
     influx_e "GRANT ALL PRIVILEGES ON telegraf TO $INFLUX_USERNAME"
+
     # create read-only user
     if ! influx -execute "show users" | grep -q "ear";
     then
         influx_e "CREATE USER ear WITH PASSWORD 'listener'"
     fi
+
     # revoke read-only user permissions
     influx_e "REVOKE ALL PRIVILEGES FROM ear"
     influx_e "GRANT READ ON multi_ear TO ear"
     influx_e "GRANT READ ON telegraf TO ear"
 
-    # enforce multi-ear settings (requires login from now on!)
-    verbose_msg "> enable multi-ear configuration" 1
-    sudo ln -sf /etc/influxdb/multi-ear.conf /etc/influxdb/influxdb.conf >> $LOG_FILE 2>&1
-    # restart service
-    do_systemd_service_restart "influxdb"
     # done
     verbose_done
 }
@@ -640,6 +669,7 @@ function do_configure
     do_configure_influxdb
     do_configure_telegraf
     do_configure_grafana
+    do_systemd_env
 }
 
 

diff --git a/multi_ear_services/__init__.py b/multi_ear_services/__init__.py
@@ -16,10 +16,6 @@
     (https://www.gnu.org/licenses/gpl-3.0.en.html)
 """
 
-from . import ctrl, uart
-
-__all__ = ['ctrl', 'uart']
-
 # Version
 try:
     # - Released versions just tags:       1.10.0

diff --git a/multi_ear_services/ctrl/__init__.py b/multi_ear_services/ctrl/__init__.py
@@ -162,9 +162,15 @@ def api_dataselect_health():
     def api_dataselect_query():
         ds = DataSelect(
             db_client,
-            starttime=request.args.get('starttime') or request.args.get('start'),
-            endtime=request.args.get('endtime') or request.args.get('end'),
-            bucket=request.args.get('bucket') or request.args.get('b'),
+            starttime=(request.args.get('starttime') or
+                       request.args.get('start') or
+                       request.args.get('s')),
+            endtime=(request.args.get('endtime') or
+                     request.args.get('end') or
+                     request.args.get('e')),
+            database=(request.args.get('database') or
+                      request.args.get('db') or
+                      request.args.get('d')),
             measurement=request.args.get('measurement') or request.args.get('m'),
             field=request.args.get('field') or request.args.get('f'),
             format=request.args.get('format') or request.args.get('_f'),

diff --git a/...ar_services/ctrl/static/css/multi_ear.css → ...ar_services/ctrl/static/css/multi-ear.css b/...ar_services/ctrl/static/css/multi_ear.css → ...ar_services/ctrl/static/css/multi-ear.css
@@ -120,7 +120,7 @@ div.pcb {
     width: 100%;
   }
   nav.navbar i {
-    padding-left: .5rem;
+    margin-left: -1.5rem;
     padding-right: .5rem;
   }
   .navbar .border-top {

diff --git a/...rvices/ctrl/static/js/vanilla-get-json.js → ...r_services/ctrl/static/js/get-response.js b/...rvices/ctrl/static/js/vanilla-get-json.js → ...r_services/ctrl/static/js/get-response.js
@@ -1,12 +1,12 @@
 /*
  * Usage:
- * getJSON("https://jsonplaceholder.typicode.com/comments", { postId: 1}, 'GET'| 'POST')
+ * getResponse("https://jsonplaceholder.typicode.com/comments", { postId: 1}, 'GET'| 'POST')
  *  .then(data => {
  *    console.log(data);
  *  });
  */
 
-function getJSON(url, qs_params, qs_type) {
+function getResponse(url, qs_params, qs_type) {
 
     function buildQueryString(params) {