redirect logs to files

deployment/.template.env

@@ -59,4 +59,11 @@ APP_DOMAIN=localhost
 # - reference_dataset
 # - machine_info_fetcher
 PLUGINS=
-# -----------------------------
+# -----------------------------
+
+# log rotate max files
+LOGROTATE_COPIES=14
+# log rotate max size per file
+LOGROTATE_SIZE=100M
+# log rotate interval check
+LOGROTATE_INTERVAL=daily

deployment/docker-compose.override.template.yml

@@ -20,6 +20,13 @@ services:
   dbbackups:
     volumes:
       - ./volumes/backups:/backups
+  redis:
+    volumes:
+      # fix redis permission issue because it's using user redis (1001)
+      # sudo chown -R 1001:1001 ./volumes/tmp_data/redis
+      - ./volumes/tmp_data/redis:/tmp
+      - ./redis/redis.conf:/opt/bitnami/redis/mounted-etc/overrides.conf
+
   django:
     build:
       context: ../
@@ -30,6 +37,8 @@ services:
       - ./volumes/static:/home/web/static
       - ./volumes/media:/home/web/media
       - ./volumes/backups:/backups
+      - ./volumes/tmp_data/django:/tmp
+      - ./volumes/tmp_data:/home/web/logs
   worker:
     build:
       context: ../
@@ -38,6 +47,7 @@ services:
       - ../django_project:/home/web/django_project
       - ./volumes/static:/home/web/static
       - ./volumes/media:/home/web/media
+      - ./volumes/tmp_data/worker:/tmp
   celery_beat:
     build:
       context: ../
@@ -46,6 +56,7 @@ services:
       - ../django_project:/home/web/django_project
       - ./volumes/static:/home/web/static
       - ./volumes/media:/home/web/media
+      - ./volumes/tmp_data/celery_beat:/tmp
   nginx:
     build:
       context: ../
@@ -54,6 +65,7 @@ services:
       - ./nginx/sites-enabled:/etc/nginx/conf.d
       - ./volumes/static:/home/web/static:ro
       - ./volumes/media:/home/web/media:ro
+      - ./volumes/tmp_data/nginx:/tmp
     ports:
       - "${HTTP_PORT:-80}:8080"
       - "${HTTPS_PORT:-443}:443"
@@ -72,6 +84,7 @@ services:
       - ./volumes/static:/home/web/static
       - ./volumes/media:/home/web/media
       - ./volumes/backups:/backups
+      - ./volumes/tmp_data:/home/web/logs
     environment:
       # editable in .env
       - DATABASE_TEMP_NAME=${DATABASE_TEMP_NAME:-temp}
@@ -113,4 +126,9 @@ services:
     links:
       - db
       - redis
-      - celery_beat
+      - celery_beat
+
+  logrotate:
+    volumes:
+      - ./volumes/tmp_data:/logs
+      - ./volumes/tmp_data/logrotate:/tmp

deployment/docker-compose.yml

@@ -20,6 +20,9 @@ volumes:
   backups-data:
   rabbitmq:
   redis-data:
+  tmp-data:
+  redis-conf:
+  tmp-logrotate:
 
 x-common-django:
   &default-common-django
@@ -66,6 +69,7 @@ x-common-django:
   volumes:
     - static-data:/home/web/static
     - media-data:/home/web/media
+    - tmp-data:/tmp
   restart: on-failure
 
 services:
@@ -76,6 +80,8 @@ services:
       - REDIS_PASSWORD=${REDIS_PASSWORD:-redis_password}
     volumes:
       - redis-data:/bitnami/redis/data
+      - tmp-data:/tmp
+      - redis-conf:/opt/bitnami/redis/mounted-etc/overrides.conf
 
   db:
     image: kartoza/postgis:13.0
@@ -121,7 +127,7 @@ services:
   worker:
     <<: *default-common-django
     container_name: "geosight_worker"
-    command: 'celery -A core worker -l info'
+    command: 'celery -A core worker -l info --logfile=/tmp/worker.log'
     entrypoint: [ ]
     links:
       - db
@@ -132,7 +138,7 @@ services:
     <<: *default-common-django
     container_name: "geosight_celery_beat"
     entrypoint: [ ]
-    command: 'celery -A core beat --loglevel=info --scheduler django_celery_beat.schedulers:DatabaseScheduler'
+    command: 'celery -A core beat --loglevel=info --scheduler django_celery_beat.schedulers:DatabaseScheduler --logfile=/tmp/celery_beat.log'
     links:
       - db
       - redis
@@ -144,6 +150,19 @@ services:
     volumes:
       - static-data:/home/web/static:ro
       - media-data:/home/web/media:ro
+      - tmp-data:/tmp
     ports:
       - "${HTTP_PORT:-80}:8080"
     restart: on-failure:5
+
+  logrotate:
+    image: blacklabelops/logrotate:1.3
+    volumes:
+      - tmp-data:/logs
+      - tmp-logrotate:/tmp
+    environment:
+      - LOGS_DIRECTORIES=/logs
+      - LOGROTATE_STATUSFILE=/tmp/logrotate.status
+      - LOGROTATE_COPIES=${LOGROTATE_COPIES:-14}
+      - LOGROTATE_SIZE=${LOGROTATE_SIZE:-100M}
+      - LOGROTATE_INTERVAL=${LOGROTATE_INTERVAL:-daily}

deployment/docker/uwsgi.conf

@@ -25,10 +25,11 @@ cheaper = 2
 env = DJANGO_SETTINGS_MODULE=core.settings.prod
 # disabled so we run in the foreground for docker
 # daemonize = /tmp/django.log
+logto = /tmp/django.log
 # reload-os-env
 # uid = 1000
 # gid = 1000
 memory-report = true
 harakiri = 2400
 buffer-size = 8192
-disable-logging = True
+# disable-logging = True

deployment/nginx/nginx.conf

@@ -1,8 +1,8 @@
 user  nginx;
 worker_processes  auto;
 
-error_log  /var/log/nginx/error.log notice;
-pid        /var/run/nginx.pid;
+error_log  /tmp/error.log notice;
+pid        /tmp/nginx.pid;
 
 events {
     worker_connections  1024;
@@ -21,7 +21,7 @@ http {
                       '$status $body_bytes_sent "$http_referer" '
                       '"$http_user_agent" "$http_x_forwarded_for"';
 
-    access_log  /var/log/nginx/access.log  main if=$logme;
+    access_log  /tmp/access.log  main if=$logme;
 
     sendfile        on;
     #tcp_nopush     on;

deployment/redis/redis.conf

@@ -0,0 +1 @@
+logfile /tmp/redis.log

django_project/core/settings/project.py

@@ -191,3 +191,9 @@
     INSTALLED_APPS = [
         app for app in INSTALLED_APPS if app != 'django.contrib.admin'
     ]
+
+# ----------------------------------------
+# Logs Directory
+# ----------------------------------------
+LOGS_DIRECTORY = os.environ.get(
+    'LOGS_DIRECTORY', '/home/web/logs')

django_project/geosight/machine_info_fetcher/admin.py

@@ -14,8 +14,108 @@
 __date__ = '22/10/2024'
 __copyright__ = ('Copyright 2023, Unicef')
 
-from django.contrib import admin
+import os
+import re
+import mimetypes
+from datetime import datetime
+from django.conf import settings
+from django.contrib import admin, messages
+from django.urls import re_path, reverse
+from django.http import (
+    HttpResponseRedirect,
+    HttpResponse,
+    Http404,
+    FileResponse
+)
+from django.utils.html import format_html
+
+from geosight.machine_info_fetcher.models import MachineInfo, LogFile
+
+
+def list_log_files(parent_dir):
+    """Get list of log files from parent_dir."""
+    log_files = []
+    for root, dirs, files in os.walk(parent_dir):
+        for file in files:
+            if ".log" in file:
+                file_path = os.path.join(root, file)
+                file_size = os.path.getsize(file_path)
+                created_on = datetime.fromtimestamp(
+                    os.path.getctime(file_path)
+                ).strftime('%Y-%m-%d %H:%M:%S')
+                log_files.append((file_path, file_size, created_on))
+    return log_files
+
+
+class LogFileAdmin(admin.ModelAdmin):
+    """Admin class for LogFile."""
+
+    list_display = (
+        'filename', 'size', 'created_on', 'download_link')
+    actions = ['refresh_log_files']
+
+    def download_link(self, obj):
+        return format_html(
+            '<a href="{}">Download</a>',
+            reverse('admin:dashboard_download_log_file',
+                    args=[obj.pk])
+        )
+    download_link.short_description = 'Download Log File'
+
+    def get_urls(self):
+        urls = super().get_urls()
+        custom_urls = [
+            re_path(r'^download-log-file/(?P<pk>\d+)$',
+                self.download_log_file,
+                name='dashboard_download_log_file'
+            ),
+        ]
+        return custom_urls + urls
+
+    def download_log_file(self, request, pk):
+        try:
+            log_file = LogFile.objects.get(pk=pk)
+            file_path = log_file.path
+            file_handle = open(file_path, 'rb')
+
+            # Use FileResponse for efficient streaming
+            response = FileResponse(
+                file_handle,
+                content_type=mimetypes.guess_type(file_path)[0]
+            )
+            response['Content-Disposition'] = (
+                f'attachment; filename="{log_file.filename()}"'
+            )
+            return response
+        except LogFile.DoesNotExist:
+            raise Http404("Log file not found")
+        except Exception as e:
+            return HttpResponse(f"Error: {e}", status=500)
+
+    def refresh_log_files(self, request, queryset):
+        """Refresh log file list."""
+        LogFile.objects.all().delete()
+
+        # Insert new log files
+        new_log_files = list_log_files(settings.LOGS_DIRECTORY)
+        log_entries = [
+            LogFile(path=path, size=size, created_on=created_on)
+            for path, size, created_on in new_log_files
+        ]
+        LogFile.objects.bulk_create(log_entries)
+
+        # Send a success message
+        self.message_user(
+            request,
+            "Log files have been refreshed.",
+            level=messages.SUCCESS
+        )
+
+        # Redirect back to the log file list page
+        return HttpResponseRedirect(request.get_full_path())
+
+    refresh_log_files.short_description = "Refresh log files"
 
-from geosight.machine_info_fetcher.models import MachineInfo
 
 admin.site.register(MachineInfo, admin.ModelAdmin)
+admin.site.register(LogFile, LogFileAdmin)

django_project/geosight/machine_info_fetcher/migrations/0002_logfile.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.2.16 on 2024-11-12 16:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('geosight_machine_info_fetcher', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='LogFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('path', models.CharField(max_length=500, unique=True)),
+                ('size', models.PositiveBigIntegerField()),
+                ('created_on', models.DateTimeField()),
+            ],
+        ),
+    ]

django_project/geosight/machine_info_fetcher/models.py

@@ -38,3 +38,18 @@ class MachineInfo(models.Model):
 
     def __str__(self):
         return f'{self.date_time}'
+
+
+class LogFile(models.Model):
+    """Class that represent log file."""
+
+    path = models.CharField(max_length=500, unique=True)
+    size = models.PositiveBigIntegerField()
+    created_on = models.DateTimeField()
+
+    def filename(self):
+        """Extracts the filename from the path."""
+        return self.path.split('/')[-1]
+
+    def __str__(self):
+        return self.filename()