Update monitoring workflow

.github/workflows/statistics.py

@@ -81,17 +81,27 @@ def filter(self, start: datetime.datetime, end: datetime.datetime) -> pd.DataFra
 
 def plot_last_month(data: Usage) -> None:
     # Get last month
-    last_month = data.filter(
-        datetime.datetime(data.today.year, data.today.month - 1, 1),
-        datetime.datetime(data.today.year, data.today.month, 1),
-    )
+    print(datetime.datetime(data.today.year, data.today.month, 1))
+    if data.today.month == 1:
+        last_month = data.filter(
+            datetime.datetime(data.today.year - 1, 12, 1),
+            datetime.datetime(data.today.year, 1, 1),
+        )
+    else:
+        last_month = data.filter(
+            datetime.datetime(data.today.year, data.today.month - 1, 1),
+            datetime.datetime(data.today.year, data.today.month, 1),
+        )
     last_month.columns = ["Jobs"]
 
     # Count usage per day from last month
     count = last_month.groupby(last_month["Jobs"].dt.day).count()
 
     # Average jobs per day
-    _, number_of_days = calendar.monthrange(data.today.year, data.today.month - 1)
+    if data.today.month == 1:
+        _, number_of_days = calendar.monthrange(data.today.year - 1, 12)
+    else:
+        _, number_of_days = calendar.monthrange(data.today.year, data.today.month - 1)
     avg = count.sum().values / number_of_days
 
     # Plot bar plot
@@ -132,16 +142,20 @@ def plot_last_month(data: Usage) -> None:
     ax.set_ylim(ymin, ymax + 1)
     ax.yaxis.set_ticks(numpy.arange(ymin, ymax + 10, 10))
     ax.tick_params(axis="y", labelsize=15)
-    ax.set_xlabel(f"{MONTHS[data.today.month-1]}-{data.today.year}", size=20)
+    if data.today.month == 1:
+        ax.set_xlabel(f"{MONTHS[12]}-{data.today.year - 1}", size=20)
+    else:
+        ax.set_xlabel(f"{MONTHS[data.today.month - 1]}-{data.today.year}", size=20)
     ax.tick_params(axis="x", labelsize=15)
     ax.xaxis.set_ticks(numpy.arange(1, number_of_days + 1, 1))
 
     # Legend
     ax.legend(loc="upper left", fontsize=20)
 
-    plt.savefig(
-        f"{MONTHS[data.today.month-1]}-{(data.today.year) % 100}-daily.png", dpi=300
-    )
+    if data.today.month == 1:
+        plt.savefig(f"{MONTHS[12]}-{(data.today.year - 1) % 100}-daily.png", dpi=300)
+    else:
+        plt.savefig(f"{MONTHS[data.today.month-1]}-{(data.today.year) % 100}-daily.png", dpi=300)
 
 
 def plot_last_12_months(data: Usage) -> None:
@@ -269,9 +283,10 @@ def plot_per_year(data: Usage) -> None:
     # Legend
     ax.legend(loc="upper left", fontsize=20)
 
-    plt.savefig(
-        f"{MONTHS[data.today.month-1]}-{data.today.year % 100}-yearly.png", dpi=300
-    )
+    if data.today.month == 1:
+        plt.savefig(f"{MONTHS[12]}-{(data.today.year - 1) % 100}-yearly.png", dpi=300)
+    else:
+        plt.savefig(f"{MONTHS[data.today.month-1]}-{data.today.year % 100}-yearly.png", dpi=300)
 
 
 if __name__ == "__main__":

.github/workflows/statistics.yml

@@ -1,13 +1,15 @@
-name: Job submission statistics
+name: KVFinder-web service job submission statistics
 on:
   schedule:
     - cron: '0 0 1 * *'
+  push:
+    branches: [master]
   workflow_dispatch:
 
 jobs:
 
   data:
-    name: Produce usage statistics
+    name: Get job submission data
     runs-on: ubuntu-latest
 
     steps:
@@ -18,14 +20,19 @@ jobs:
         id: date
         run: echo "::set-output name=date::$(date +'%Y-%m')"
 
-      - name: Get job submission data (usage.txt)
+      - name: Get data from the server
         uses: appleboy/[email protected]
         with:
           host: ${{ secrets.HOST }}
           username: ${{ secrets.USERNAME }}
           password: ${{ secrets.PASSWORD }}
-          script: | 
-            echo ${{ secrets.PASSWORD }} | sudo -S su - -c "ls /var/lib/containers/storage/volumes/kvfinder_web_service_kvfinder-jobs/_data -lhrt --time-style=long-iso >> /home/${{ secrets.USERNAME }}/usage.txt"
+          script: |
+            echo ${{ secrets.PASSWORD }} | sudo -S su - -c "cat /home/${{ secrets.USERNAME }}/01_2023_to_12_2024.txt > /home/${{ secrets.USERNAME }}/usage.txt && ls /var/lib/containers/storage/volumes/kvfinder_web_service_kvfinder-jobs/_data -lhrt --time-style=long-iso >> /home/${{ secrets.USERNAME }}/usage.txt"
+
+  scp-copy:
+    name: Copy job submission data
+    runs-on: ubuntu-latest
+    needs: [data]
 
       - name: Copy job submission data
         uses: nicklasfrahm/scp-action@main
@@ -37,7 +44,7 @@ jobs:
           insecure_ignore_fingerprint: true
           source: /home/${{ secrets.USERNAME }}/usage.txt
           target: ./usage.txt
-      
+
       - name: Run python script to produce usage plots
         run: |
           pip install -r .github/workflows/requirements.txt
@@ -70,4 +77,4 @@ jobs:
           username: ${{ secrets.USERNAME }}
           password: ${{ secrets.PASSWORD }}
           script: | 
-            echo ${{ secrets.PASSWORD }} | sudo -S su - -c "rm /home/${{ secrets.USERNAME }}/usage.txt"
+            echo ${{ secrets.PASSWORD }} | sudo -S su - -c "rm /home/${{ secrets.USERNAME }}/usage.txt"