From a4f1a0876765a21607db02e6ce838364424b9f88 Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Tue, 12 Nov 2024 16:24:05 +0000 Subject: [PATCH 1/4] redirect logs to files --- deployment/.template.env | 9 +- .../docker-compose.override.template.yml | 20 +++- deployment/docker-compose.yml | 23 +++- deployment/docker/uwsgi.conf | 3 +- deployment/nginx/nginx.conf | 6 +- deployment/redis/redis.conf | 1 + django_project/core/settings/project.py | 6 + .../geosight/machine_info_fetcher/admin.py | 104 +++++++++++++++++- .../migrations/0002_logfile.py | 22 ++++ .../geosight/machine_info_fetcher/models.py | 15 +++ 10 files changed, 199 insertions(+), 10 deletions(-) create mode 100644 deployment/redis/redis.conf create mode 100644 django_project/geosight/machine_info_fetcher/migrations/0002_logfile.py diff --git a/deployment/.template.env b/deployment/.template.env index a1b4624f1..9da531a5f 100644 --- a/deployment/.template.env +++ b/deployment/.template.env @@ -59,4 +59,11 @@ APP_DOMAIN=localhost # - reference_dataset # - machine_info_fetcher PLUGINS= -# ----------------------------- \ No newline at end of file +# ----------------------------- + +# log rotate max files +LOGROTATE_COPIES=14 +# log rotate max size per file +LOGROTATE_SIZE=100M +# log rotate interval check +LOGROTATE_INTERVAL=daily diff --git a/deployment/docker-compose.override.template.yml b/deployment/docker-compose.override.template.yml index b2dcfc5cf..4fb2a0c73 100644 --- a/deployment/docker-compose.override.template.yml +++ b/deployment/docker-compose.override.template.yml @@ -20,6 +20,13 @@ services: dbbackups: volumes: - ./volumes/backups:/backups + redis: + volumes: + # fix redis permission issue because it's using user redis (1001) + # sudo chown -R 1001:1001 ./volumes/tmp_data/redis + - ./volumes/tmp_data/redis:/tmp + - ./redis/redis.conf:/opt/bitnami/redis/mounted-etc/overrides.conf + django: build: context: ../ @@ -30,6 +37,8 @@ services: - ./volumes/static:/home/web/static - ./volumes/media:/home/web/media - ./volumes/backups:/backups + - ./volumes/tmp_data/django:/tmp + - ./volumes/tmp_data:/home/web/logs worker: build: context: ../ @@ -38,6 +47,7 @@ services: - ../django_project:/home/web/django_project - ./volumes/static:/home/web/static - ./volumes/media:/home/web/media + - ./volumes/tmp_data/worker:/tmp celery_beat: build: context: ../ @@ -46,6 +56,7 @@ services: - ../django_project:/home/web/django_project - ./volumes/static:/home/web/static - ./volumes/media:/home/web/media + - ./volumes/tmp_data/celery_beat:/tmp nginx: build: context: ../ @@ -54,6 +65,7 @@ services: - ./nginx/sites-enabled:/etc/nginx/conf.d - ./volumes/static:/home/web/static:ro - ./volumes/media:/home/web/media:ro + - ./volumes/tmp_data/nginx:/tmp ports: - "${HTTP_PORT:-80}:8080" - "${HTTPS_PORT:-443}:443" @@ -72,6 +84,7 @@ services: - ./volumes/static:/home/web/static - ./volumes/media:/home/web/media - ./volumes/backups:/backups + - ./volumes/tmp_data:/home/web/logs environment: # editable in .env - DATABASE_TEMP_NAME=${DATABASE_TEMP_NAME:-temp} @@ -113,4 +126,9 @@ services: links: - db - redis - - celery_beat \ No newline at end of file + - celery_beat + + logrotate: + volumes: + - ./volumes/tmp_data:/logs + - ./volumes/tmp_data/logrotate:/tmp diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index e3ad0ca6b..e14c06752 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -20,6 +20,9 @@ volumes: backups-data: rabbitmq: redis-data: + tmp-data: + redis-conf: + tmp-logrotate: x-common-django: &default-common-django @@ -66,6 +69,7 @@ x-common-django: volumes: - static-data:/home/web/static - media-data:/home/web/media + - tmp-data:/tmp restart: on-failure services: @@ -76,6 +80,8 @@ services: - REDIS_PASSWORD=${REDIS_PASSWORD:-redis_password} volumes: - redis-data:/bitnami/redis/data + - tmp-data:/tmp + - redis-conf:/opt/bitnami/redis/mounted-etc/overrides.conf db: image: kartoza/postgis:13.0 @@ -121,7 +127,7 @@ services: worker: <<: *default-common-django container_name: "geosight_worker" - command: 'celery -A core worker -l info' + command: 'celery -A core worker -l info --logfile=/tmp/worker.log' entrypoint: [ ] links: - db @@ -132,7 +138,7 @@ services: <<: *default-common-django container_name: "geosight_celery_beat" entrypoint: [ ] - command: 'celery -A core beat --loglevel=info --scheduler django_celery_beat.schedulers:DatabaseScheduler' + command: 'celery -A core beat --loglevel=info --scheduler django_celery_beat.schedulers:DatabaseScheduler --logfile=/tmp/celery_beat.log' links: - db - redis @@ -144,6 +150,19 @@ services: volumes: - static-data:/home/web/static:ro - media-data:/home/web/media:ro + - tmp-data:/tmp ports: - "${HTTP_PORT:-80}:8080" restart: on-failure:5 + + logrotate: + image: blacklabelops/logrotate:1.3 + volumes: + - tmp-data:/logs + - tmp-logrotate:/tmp + environment: + - LOGS_DIRECTORIES=/logs + - LOGROTATE_STATUSFILE=/tmp/logrotate.status + - LOGROTATE_COPIES=${LOGROTATE_COPIES:-14} + - LOGROTATE_SIZE=${LOGROTATE_SIZE:-100M} + - LOGROTATE_INTERVAL=${LOGROTATE_INTERVAL:-daily} diff --git a/deployment/docker/uwsgi.conf b/deployment/docker/uwsgi.conf index 848e51c97..696f213b5 100644 --- a/deployment/docker/uwsgi.conf +++ b/deployment/docker/uwsgi.conf @@ -25,10 +25,11 @@ cheaper = 2 env = DJANGO_SETTINGS_MODULE=core.settings.prod # disabled so we run in the foreground for docker # daemonize = /tmp/django.log +logto = /tmp/django.log # reload-os-env # uid = 1000 # gid = 1000 memory-report = true harakiri = 2400 buffer-size = 8192 -disable-logging = True \ No newline at end of file +# disable-logging = True \ No newline at end of file diff --git a/deployment/nginx/nginx.conf b/deployment/nginx/nginx.conf index a875cc5d6..b9549848f 100644 --- a/deployment/nginx/nginx.conf +++ b/deployment/nginx/nginx.conf @@ -1,8 +1,8 @@ user nginx; worker_processes auto; -error_log /var/log/nginx/error.log notice; -pid /var/run/nginx.pid; +error_log /tmp/error.log notice; +pid /tmp/nginx.pid; events { worker_connections 1024; @@ -21,7 +21,7 @@ http { '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main if=$logme; + access_log /tmp/access.log main if=$logme; sendfile on; #tcp_nopush on; diff --git a/deployment/redis/redis.conf b/deployment/redis/redis.conf new file mode 100644 index 000000000..c2bf80880 --- /dev/null +++ b/deployment/redis/redis.conf @@ -0,0 +1 @@ +logfile /tmp/redis.log \ No newline at end of file diff --git a/django_project/core/settings/project.py b/django_project/core/settings/project.py index ee926a811..df0fafa9b 100644 --- a/django_project/core/settings/project.py +++ b/django_project/core/settings/project.py @@ -191,3 +191,9 @@ INSTALLED_APPS = [ app for app in INSTALLED_APPS if app != 'django.contrib.admin' ] + +# ---------------------------------------- +# Logs Directory +# ---------------------------------------- +LOGS_DIRECTORY = os.environ.get( + 'LOGS_DIRECTORY', '/home/web/logs') diff --git a/django_project/geosight/machine_info_fetcher/admin.py b/django_project/geosight/machine_info_fetcher/admin.py index 001ab8a48..acbd8c0ca 100644 --- a/django_project/geosight/machine_info_fetcher/admin.py +++ b/django_project/geosight/machine_info_fetcher/admin.py @@ -14,8 +14,108 @@ __date__ = '22/10/2024' __copyright__ = ('Copyright 2023, Unicef') -from django.contrib import admin +import os +import re +import mimetypes +from datetime import datetime +from django.conf import settings +from django.contrib import admin, messages +from django.urls import re_path, reverse +from django.http import ( + HttpResponseRedirect, + HttpResponse, + Http404, + FileResponse +) +from django.utils.html import format_html + +from geosight.machine_info_fetcher.models import MachineInfo, LogFile + + +def list_log_files(parent_dir): + """Get list of log files from parent_dir.""" + log_files = [] + for root, dirs, files in os.walk(parent_dir): + for file in files: + if ".log" in file: + file_path = os.path.join(root, file) + file_size = os.path.getsize(file_path) + created_on = datetime.fromtimestamp( + os.path.getctime(file_path) + ).strftime('%Y-%m-%d %H:%M:%S') + log_files.append((file_path, file_size, created_on)) + return log_files + + +class LogFileAdmin(admin.ModelAdmin): + """Admin class for LogFile.""" + + list_display = ( + 'filename', 'size', 'created_on', 'download_link') + actions = ['refresh_log_files'] + + def download_link(self, obj): + return format_html( + 'Download', + reverse('admin:dashboard_download_log_file', + args=[obj.pk]) + ) + download_link.short_description = 'Download Log File' + + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + re_path(r'^download-log-file/(?P\d+)$', + self.download_log_file, + name='dashboard_download_log_file' + ), + ] + return custom_urls + urls + + def download_log_file(self, request, pk): + try: + log_file = LogFile.objects.get(pk=pk) + file_path = log_file.path + file_handle = open(file_path, 'rb') + + # Use FileResponse for efficient streaming + response = FileResponse( + file_handle, + content_type=mimetypes.guess_type(file_path)[0] + ) + response['Content-Disposition'] = ( + f'attachment; filename="{log_file.filename()}"' + ) + return response + except LogFile.DoesNotExist: + raise Http404("Log file not found") + except Exception as e: + return HttpResponse(f"Error: {e}", status=500) + + def refresh_log_files(self, request, queryset): + """Refresh log file list.""" + LogFile.objects.all().delete() + + # Insert new log files + new_log_files = list_log_files(settings.LOGS_DIRECTORY) + log_entries = [ + LogFile(path=path, size=size, created_on=created_on) + for path, size, created_on in new_log_files + ] + LogFile.objects.bulk_create(log_entries) + + # Send a success message + self.message_user( + request, + "Log files have been refreshed.", + level=messages.SUCCESS + ) + + # Redirect back to the log file list page + return HttpResponseRedirect(request.get_full_path()) + + refresh_log_files.short_description = "Refresh log files" -from geosight.machine_info_fetcher.models import MachineInfo admin.site.register(MachineInfo, admin.ModelAdmin) +admin.site.register(LogFile, LogFileAdmin) diff --git a/django_project/geosight/machine_info_fetcher/migrations/0002_logfile.py b/django_project/geosight/machine_info_fetcher/migrations/0002_logfile.py new file mode 100644 index 000000000..cd250ef90 --- /dev/null +++ b/django_project/geosight/machine_info_fetcher/migrations/0002_logfile.py @@ -0,0 +1,22 @@ +# Generated by Django 3.2.16 on 2024-11-12 16:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('geosight_machine_info_fetcher', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='LogFile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('path', models.CharField(max_length=500, unique=True)), + ('size', models.PositiveBigIntegerField()), + ('created_on', models.DateTimeField()), + ], + ), + ] diff --git a/django_project/geosight/machine_info_fetcher/models.py b/django_project/geosight/machine_info_fetcher/models.py index 42060eb0f..5fd266c62 100644 --- a/django_project/geosight/machine_info_fetcher/models.py +++ b/django_project/geosight/machine_info_fetcher/models.py @@ -38,3 +38,18 @@ class MachineInfo(models.Model): def __str__(self): return f'{self.date_time}' + + +class LogFile(models.Model): + """Class that represent log file.""" + + path = models.CharField(max_length=500, unique=True) + size = models.PositiveBigIntegerField() + created_on = models.DateTimeField() + + def filename(self): + """Extracts the filename from the path.""" + return self.path.split('/')[-1] + + def __str__(self): + return self.filename() From 3eb0895f09fc537f7197faf22c7032f6fcf2543b Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Tue, 12 Nov 2024 17:04:16 +0000 Subject: [PATCH 2/4] fix lint --- django_project/geosight/machine_info_fetcher/admin.py | 9 ++++++--- django_project/geosight/machine_info_fetcher/models.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/django_project/geosight/machine_info_fetcher/admin.py b/django_project/geosight/machine_info_fetcher/admin.py index acbd8c0ca..242421492 100644 --- a/django_project/geosight/machine_info_fetcher/admin.py +++ b/django_project/geosight/machine_info_fetcher/admin.py @@ -15,7 +15,6 @@ __copyright__ = ('Copyright 2023, Unicef') import os -import re import mimetypes from datetime import datetime from django.conf import settings @@ -48,13 +47,14 @@ def list_log_files(parent_dir): class LogFileAdmin(admin.ModelAdmin): - """Admin class for LogFile.""" + """Class that represents LogFile Admin.""" list_display = ( 'filename', 'size', 'created_on', 'download_link') actions = ['refresh_log_files'] def download_link(self, obj): + """Get download link for LogFile.""" return format_html( 'Download', reverse('admin:dashboard_download_log_file', @@ -63,9 +63,11 @@ def download_link(self, obj): download_link.short_description = 'Download Log File' def get_urls(self): + """Add url for download link LogFile.""" urls = super().get_urls() custom_urls = [ - re_path(r'^download-log-file/(?P\d+)$', + re_path( + r'^download-log-file/(?P\d+)$', self.download_log_file, name='dashboard_download_log_file' ), @@ -73,6 +75,7 @@ def get_urls(self): return custom_urls + urls def download_log_file(self, request, pk): + """Download log file action.""" try: log_file = LogFile.objects.get(pk=pk) file_path = log_file.path diff --git a/django_project/geosight/machine_info_fetcher/models.py b/django_project/geosight/machine_info_fetcher/models.py index 5fd266c62..90fa3730a 100644 --- a/django_project/geosight/machine_info_fetcher/models.py +++ b/django_project/geosight/machine_info_fetcher/models.py @@ -48,7 +48,7 @@ class LogFile(models.Model): created_on = models.DateTimeField() def filename(self): - """Extracts the filename from the path.""" + """Get the filename from the path.""" return self.path.split('/')[-1] def __str__(self): From 4d2178832ead4dbf284c0808f38da4eb25cc5e10 Mon Sep 17 00:00:00 2001 From: Irwan Fathurrahman Date: Wed, 13 Nov 2024 10:15:25 +0700 Subject: [PATCH 3/4] Move the files to new app --- .../migrations/0035_logfile.py} | 4 +- .../core/migrations/0036_delete_logfile.py | 16 +++ django_project/core/settings/apps/project.py | 1 + django_project/entrypoint.sh | 10 +- .../migrations/0113_indicatorvaluedataset.py | 33 +++++ django_project/geosight/log/__init__.py | 28 ++++ django_project/geosight/log/admin.py | 123 ++++++++++++++++++ .../geosight/log/migrations/0001_initial.py | 23 ++++ .../geosight/log/migrations/__init__.py | 0 django_project/geosight/log/models.py | 36 +++++ .../geosight/machine_info_fetcher/admin.py | 107 +-------------- .../geosight/machine_info_fetcher/models.py | 15 --- 12 files changed, 269 insertions(+), 127 deletions(-) rename django_project/{geosight/machine_info_fetcher/migrations/0002_logfile.py => core/migrations/0035_logfile.py} (82%) create mode 100644 django_project/core/migrations/0036_delete_logfile.py create mode 100644 django_project/geosight/data/migrations/0113_indicatorvaluedataset.py create mode 100644 django_project/geosight/log/__init__.py create mode 100644 django_project/geosight/log/admin.py create mode 100644 django_project/geosight/log/migrations/0001_initial.py create mode 100644 django_project/geosight/log/migrations/__init__.py create mode 100644 django_project/geosight/log/models.py diff --git a/django_project/geosight/machine_info_fetcher/migrations/0002_logfile.py b/django_project/core/migrations/0035_logfile.py similarity index 82% rename from django_project/geosight/machine_info_fetcher/migrations/0002_logfile.py rename to django_project/core/migrations/0035_logfile.py index cd250ef90..67f87dc19 100644 --- a/django_project/geosight/machine_info_fetcher/migrations/0002_logfile.py +++ b/django_project/core/migrations/0035_logfile.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.16 on 2024-11-12 16:15 +# Generated by Django 3.2.16 on 2024-11-13 02:40 from django.db import migrations, models @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('geosight_machine_info_fetcher', '0001_initial'), + ('core', '0034_sitepreferences_machine_info_fetcher_config'), ] operations = [ diff --git a/django_project/core/migrations/0036_delete_logfile.py b/django_project/core/migrations/0036_delete_logfile.py new file mode 100644 index 000000000..a10919951 --- /dev/null +++ b/django_project/core/migrations/0036_delete_logfile.py @@ -0,0 +1,16 @@ +# Generated by Django 3.2.16 on 2024-11-13 02:46 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0035_logfile'), + ] + + operations = [ + migrations.DeleteModel( + name='LogFile', + ), + ] diff --git a/django_project/core/settings/apps/project.py b/django_project/core/settings/apps/project.py index 595da90f0..fab901699 100644 --- a/django_project/core/settings/apps/project.py +++ b/django_project/core/settings/apps/project.py @@ -23,5 +23,6 @@ 'geosight.georepo', 'geosight.permission', 'geosight.importer', + 'geosight.log', 'frontend' ] diff --git a/django_project/entrypoint.sh b/django_project/entrypoint.sh index b4399c8b7..c16cde585 100755 --- a/django_project/entrypoint.sh +++ b/django_project/entrypoint.sh @@ -22,11 +22,11 @@ echo "STARTING DJANGO ENTRYPOINT $(date)" echo "-----------------------------------------------------" # Run NPM -cd /home/web/django_project/frontend -echo "npm install" -npm install --verbose -echo "npm build" -npm run build +#cd /home/web/django_project/frontend +#echo "npm install" +#npm install --verbose +#echo "npm build" +#npm run build # Run initialization cd /home/web/django_project diff --git a/django_project/geosight/data/migrations/0113_indicatorvaluedataset.py b/django_project/geosight/data/migrations/0113_indicatorvaluedataset.py new file mode 100644 index 000000000..c97d9b99b --- /dev/null +++ b/django_project/geosight/data/migrations/0113_indicatorvaluedataset.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.16 on 2024-11-13 02:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('geosight_data', '0112_alter_context_layer'), + ] + + operations = [ + migrations.CreateModel( + name='IndicatorValueDataset', + fields=[ + ('id', models.CharField(max_length=256, primary_key=True, serialize=False)), + ('indicator_id', models.BigIntegerField()), + ('reference_layer_id', models.UUIDField()), + ('admin_level', models.CharField(max_length=256)), + ('data_count', models.IntegerField(blank=True, null=True)), + ('start_date', models.DateField(blank=True, null=True)), + ('end_date', models.DateField(blank=True, null=True)), + ('reference_layer_uuid', models.UUIDField()), + ('reference_layer_name', models.CharField(blank=True, max_length=256, null=True)), + ('indicator_name', models.CharField(blank=True, max_length=256, null=True)), + ('indicator_shortcode', models.CharField(blank=True, max_length=256, null=True)), + ('identifier', models.CharField(blank=True, max_length=256, null=True)), + ], + options={ + 'managed': False, + }, + ), + ] diff --git a/django_project/geosight/log/__init__.py b/django_project/geosight/log/__init__.py new file mode 100644 index 000000000..75fe0a791 --- /dev/null +++ b/django_project/geosight/log/__init__.py @@ -0,0 +1,28 @@ +# coding=utf-8 +""" +GeoSight is UNICEF's geospatial web-based business intelligence platform. + +Contact : geosight-no-reply@unicef.org + +.. note:: This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +""" +__author__ = 'irwan@kartoza.com' +__date__ = '13/11/2024' +__copyright__ = ('Copyright 2023, Unicef') + +from django.apps import AppConfig + + +class Config(AppConfig): + """Machine log app.""" + + label = 'geosight_log' + name = 'geosight.log' + verbose_name = "GeoSight Log" + + +default_app_config = 'geosight.log.Config' diff --git a/django_project/geosight/log/admin.py b/django_project/geosight/log/admin.py new file mode 100644 index 000000000..102ee00c7 --- /dev/null +++ b/django_project/geosight/log/admin.py @@ -0,0 +1,123 @@ +# coding=utf-8 +""" +GeoSight is UNICEF's geospatial web-based business intelligence platform. + +Contact : geosight-no-reply@unicef.org + +.. note:: This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +""" +__author__ = 'irwan@kartoza.com' +__date__ = '22/10/2024' +__copyright__ = ('Copyright 2023, Unicef') + +import mimetypes +import os +from datetime import datetime + +from django.conf import settings +from django.contrib import admin, messages +from django.http import ( + HttpResponseRedirect, + HttpResponse, + Http404, + FileResponse +) +from django.urls import re_path, reverse +from django.utils.html import format_html + +from geosight.log.models import LogFile + + +def list_log_files(parent_dir): + """Get list of log files from parent_dir.""" + log_files = [] + for root, dirs, files in os.walk(parent_dir): + for file in files: + if ".log" in file: + file_path = os.path.join(root, file) + file_size = os.path.getsize(file_path) + created_on = datetime.fromtimestamp( + os.path.getctime(file_path) + ).strftime('%Y-%m-%d %H:%M:%S') + log_files.append((file_path, file_size, created_on)) + return log_files + + +@admin.register(LogFile) +class LogFileAdmin(admin.ModelAdmin): + """Class that represents LogFile Admin.""" + + list_display = ( + 'filename', 'size', 'created_on', 'download_link') + actions = ['refresh_log_files'] + + def download_link(self, obj): + """Get download link for LogFile.""" + return format_html( + 'Download', + reverse('admin:dashboard_download_log_file', + args=[obj.pk]) + ) + + download_link.short_description = 'Download Log File' + + def get_urls(self): + """Add url for download link LogFile.""" + urls = super().get_urls() + custom_urls = [ + re_path( + r'^download-log-file/(?P\d+)$', + self.download_log_file, + name='dashboard_download_log_file' + ), + ] + return custom_urls + urls + + def download_log_file(self, request, pk): + """Download log file action.""" + try: + log_file = LogFile.objects.get(pk=pk) + file_path = log_file.path + file_handle = open(file_path, 'rb') + + # Use FileResponse for efficient streaming + response = FileResponse( + file_handle, + content_type=mimetypes.guess_type(file_path)[0] + ) + response['Content-Disposition'] = ( + f'attachment; filename="{log_file.filename()}"' + ) + return response + except LogFile.DoesNotExist: + raise Http404("Log file not found") + except Exception as e: + return HttpResponse(f"Error: {e}", status=500) + + def refresh_log_files(self, request, queryset): + """Refresh log file list.""" + LogFile.objects.all().delete() + + # Insert new log files + new_log_files = list_log_files(settings.LOGS_DIRECTORY) + log_entries = [ + LogFile(path=path, size=size, created_on=created_on) + for path, size, created_on in new_log_files + ] + LogFile.objects.bulk_create(log_entries) + + # Send a success message + self.message_user( + request, + "Log files have been refreshed.", + level=messages.SUCCESS + ) + + # Redirect back to the log file list page + return HttpResponseRedirect(request.get_full_path()) + + refresh_log_files.short_description = "Refresh log files" diff --git a/django_project/geosight/log/migrations/0001_initial.py b/django_project/geosight/log/migrations/0001_initial.py new file mode 100644 index 000000000..4ec880bbc --- /dev/null +++ b/django_project/geosight/log/migrations/0001_initial.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.16 on 2024-11-13 02:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='LogFile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('path', models.CharField(max_length=500, unique=True)), + ('size', models.PositiveBigIntegerField(blank=True, editable=False, null=True)), + ('created_on', models.DateTimeField(auto_now_add=True)), + ], + ), + ] diff --git a/django_project/geosight/log/migrations/__init__.py b/django_project/geosight/log/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/django_project/geosight/log/models.py b/django_project/geosight/log/models.py new file mode 100644 index 000000000..8e3749a72 --- /dev/null +++ b/django_project/geosight/log/models.py @@ -0,0 +1,36 @@ +# coding=utf-8 +""" +GeoSight is UNICEF's geospatial web-based business intelligence platform. + +Contact : geosight-no-reply@unicef.org + +.. note:: This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +""" +__author__ = 'danang@kartoza.com' +__date__ = '13/11/2024' +__copyright__ = ('Copyright 2023, Unicef') + +from django.contrib.gis.db import models + + +class LogFile(models.Model): + """Class that represent log file.""" + + path = models.CharField(max_length=500, unique=True) + size = models.PositiveBigIntegerField( + null=True, blank=True, editable=False + ) + created_on = models.DateTimeField( + auto_now_add=True, + ) + + def filename(self): + """Get the filename from the path.""" + return self.path.split('/')[-1] + + def __str__(self): + return self.filename() diff --git a/django_project/geosight/machine_info_fetcher/admin.py b/django_project/geosight/machine_info_fetcher/admin.py index 242421492..001ab8a48 100644 --- a/django_project/geosight/machine_info_fetcher/admin.py +++ b/django_project/geosight/machine_info_fetcher/admin.py @@ -14,111 +14,8 @@ __date__ = '22/10/2024' __copyright__ = ('Copyright 2023, Unicef') -import os -import mimetypes -from datetime import datetime -from django.conf import settings -from django.contrib import admin, messages -from django.urls import re_path, reverse -from django.http import ( - HttpResponseRedirect, - HttpResponse, - Http404, - FileResponse -) -from django.utils.html import format_html - -from geosight.machine_info_fetcher.models import MachineInfo, LogFile - - -def list_log_files(parent_dir): - """Get list of log files from parent_dir.""" - log_files = [] - for root, dirs, files in os.walk(parent_dir): - for file in files: - if ".log" in file: - file_path = os.path.join(root, file) - file_size = os.path.getsize(file_path) - created_on = datetime.fromtimestamp( - os.path.getctime(file_path) - ).strftime('%Y-%m-%d %H:%M:%S') - log_files.append((file_path, file_size, created_on)) - return log_files - - -class LogFileAdmin(admin.ModelAdmin): - """Class that represents LogFile Admin.""" - - list_display = ( - 'filename', 'size', 'created_on', 'download_link') - actions = ['refresh_log_files'] - - def download_link(self, obj): - """Get download link for LogFile.""" - return format_html( - 'Download', - reverse('admin:dashboard_download_log_file', - args=[obj.pk]) - ) - download_link.short_description = 'Download Log File' - - def get_urls(self): - """Add url for download link LogFile.""" - urls = super().get_urls() - custom_urls = [ - re_path( - r'^download-log-file/(?P\d+)$', - self.download_log_file, - name='dashboard_download_log_file' - ), - ] - return custom_urls + urls - - def download_log_file(self, request, pk): - """Download log file action.""" - try: - log_file = LogFile.objects.get(pk=pk) - file_path = log_file.path - file_handle = open(file_path, 'rb') - - # Use FileResponse for efficient streaming - response = FileResponse( - file_handle, - content_type=mimetypes.guess_type(file_path)[0] - ) - response['Content-Disposition'] = ( - f'attachment; filename="{log_file.filename()}"' - ) - return response - except LogFile.DoesNotExist: - raise Http404("Log file not found") - except Exception as e: - return HttpResponse(f"Error: {e}", status=500) - - def refresh_log_files(self, request, queryset): - """Refresh log file list.""" - LogFile.objects.all().delete() - - # Insert new log files - new_log_files = list_log_files(settings.LOGS_DIRECTORY) - log_entries = [ - LogFile(path=path, size=size, created_on=created_on) - for path, size, created_on in new_log_files - ] - LogFile.objects.bulk_create(log_entries) - - # Send a success message - self.message_user( - request, - "Log files have been refreshed.", - level=messages.SUCCESS - ) - - # Redirect back to the log file list page - return HttpResponseRedirect(request.get_full_path()) - - refresh_log_files.short_description = "Refresh log files" +from django.contrib import admin +from geosight.machine_info_fetcher.models import MachineInfo admin.site.register(MachineInfo, admin.ModelAdmin) -admin.site.register(LogFile, LogFileAdmin) diff --git a/django_project/geosight/machine_info_fetcher/models.py b/django_project/geosight/machine_info_fetcher/models.py index 90fa3730a..42060eb0f 100644 --- a/django_project/geosight/machine_info_fetcher/models.py +++ b/django_project/geosight/machine_info_fetcher/models.py @@ -38,18 +38,3 @@ class MachineInfo(models.Model): def __str__(self): return f'{self.date_time}' - - -class LogFile(models.Model): - """Class that represent log file.""" - - path = models.CharField(max_length=500, unique=True) - size = models.PositiveBigIntegerField() - created_on = models.DateTimeField() - - def filename(self): - """Get the filename from the path.""" - return self.path.split('/')[-1] - - def __str__(self): - return self.filename() From 4ca99b55b8d2e0661a8bf69f1f6b6cf7aa59a598 Mon Sep 17 00:00:00 2001 From: Irwan Fathurrahman Date: Wed, 13 Nov 2024 10:22:23 +0700 Subject: [PATCH 4/4] Clean code --- deployment/.template.env | 16 +++++---- deployment/docker-compose.yml | 3 +- .../core/migrations/0035_logfile.py | 22 ------------- .../core/migrations/0036_delete_logfile.py | 16 --------- django_project/core/settings/project.py | 3 +- django_project/entrypoint.sh | 10 +++--- .../migrations/0113_indicatorvaluedataset.py | 33 ------------------- django_project/geosight/log/admin.py | 6 ++-- 8 files changed, 22 insertions(+), 87 deletions(-) delete mode 100644 django_project/core/migrations/0035_logfile.py delete mode 100644 django_project/core/migrations/0036_delete_logfile.py delete mode 100644 django_project/geosight/data/migrations/0113_indicatorvaluedataset.py diff --git a/deployment/.template.env b/deployment/.template.env index 9da531a5f..426a1ccc4 100644 --- a/deployment/.template.env +++ b/deployment/.template.env @@ -43,6 +43,15 @@ EMAIL_USE_TLS= EMAIL_USE_SSL= DEFAULT_FROM_EMAIL= +# ------------------------------- +# ---------- LOG ROTATE ------------ +# log rotate max files +LOGROTATE_COPIES=14 +# log rotate max size per file +LOGROTATE_SIZE=100M +# log rotate interval check +LOGROTATE_INTERVAL=daily + # -------------------------------- # ---------- APP DOMAIN ---------- # Required for tenant configurations @@ -60,10 +69,3 @@ APP_DOMAIN=localhost # - machine_info_fetcher PLUGINS= # ----------------------------- - -# log rotate max files -LOGROTATE_COPIES=14 -# log rotate max size per file -LOGROTATE_SIZE=100M -# log rotate interval check -LOGROTATE_INTERVAL=daily diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index e14c06752..089297b9f 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -20,8 +20,8 @@ volumes: backups-data: rabbitmq: redis-data: - tmp-data: redis-conf: + tmp-data: tmp-logrotate: x-common-django: @@ -157,6 +157,7 @@ services: logrotate: image: blacklabelops/logrotate:1.3 + container_name: "geosight_logrotate" volumes: - tmp-data:/logs - tmp-logrotate:/tmp diff --git a/django_project/core/migrations/0035_logfile.py b/django_project/core/migrations/0035_logfile.py deleted file mode 100644 index 67f87dc19..000000000 --- a/django_project/core/migrations/0035_logfile.py +++ /dev/null @@ -1,22 +0,0 @@ -# Generated by Django 3.2.16 on 2024-11-13 02:40 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0034_sitepreferences_machine_info_fetcher_config'), - ] - - operations = [ - migrations.CreateModel( - name='LogFile', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('path', models.CharField(max_length=500, unique=True)), - ('size', models.PositiveBigIntegerField()), - ('created_on', models.DateTimeField()), - ], - ), - ] diff --git a/django_project/core/migrations/0036_delete_logfile.py b/django_project/core/migrations/0036_delete_logfile.py deleted file mode 100644 index a10919951..000000000 --- a/django_project/core/migrations/0036_delete_logfile.py +++ /dev/null @@ -1,16 +0,0 @@ -# Generated by Django 3.2.16 on 2024-11-13 02:46 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0035_logfile'), - ] - - operations = [ - migrations.DeleteModel( - name='LogFile', - ), - ] diff --git a/django_project/core/settings/project.py b/django_project/core/settings/project.py index df0fafa9b..c9507c8e4 100644 --- a/django_project/core/settings/project.py +++ b/django_project/core/settings/project.py @@ -196,4 +196,5 @@ # Logs Directory # ---------------------------------------- LOGS_DIRECTORY = os.environ.get( - 'LOGS_DIRECTORY', '/home/web/logs') + 'LOGS_DIRECTORY', '/home/web/logs' +) diff --git a/django_project/entrypoint.sh b/django_project/entrypoint.sh index c16cde585..b4399c8b7 100755 --- a/django_project/entrypoint.sh +++ b/django_project/entrypoint.sh @@ -22,11 +22,11 @@ echo "STARTING DJANGO ENTRYPOINT $(date)" echo "-----------------------------------------------------" # Run NPM -#cd /home/web/django_project/frontend -#echo "npm install" -#npm install --verbose -#echo "npm build" -#npm run build +cd /home/web/django_project/frontend +echo "npm install" +npm install --verbose +echo "npm build" +npm run build # Run initialization cd /home/web/django_project diff --git a/django_project/geosight/data/migrations/0113_indicatorvaluedataset.py b/django_project/geosight/data/migrations/0113_indicatorvaluedataset.py deleted file mode 100644 index c97d9b99b..000000000 --- a/django_project/geosight/data/migrations/0113_indicatorvaluedataset.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 3.2.16 on 2024-11-13 02:46 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('geosight_data', '0112_alter_context_layer'), - ] - - operations = [ - migrations.CreateModel( - name='IndicatorValueDataset', - fields=[ - ('id', models.CharField(max_length=256, primary_key=True, serialize=False)), - ('indicator_id', models.BigIntegerField()), - ('reference_layer_id', models.UUIDField()), - ('admin_level', models.CharField(max_length=256)), - ('data_count', models.IntegerField(blank=True, null=True)), - ('start_date', models.DateField(blank=True, null=True)), - ('end_date', models.DateField(blank=True, null=True)), - ('reference_layer_uuid', models.UUIDField()), - ('reference_layer_name', models.CharField(blank=True, max_length=256, null=True)), - ('indicator_name', models.CharField(blank=True, max_length=256, null=True)), - ('indicator_shortcode', models.CharField(blank=True, max_length=256, null=True)), - ('identifier', models.CharField(blank=True, max_length=256, null=True)), - ], - options={ - 'managed': False, - }, - ), - ] diff --git a/django_project/geosight/log/admin.py b/django_project/geosight/log/admin.py index 102ee00c7..640679710 100644 --- a/django_project/geosight/log/admin.py +++ b/django_project/geosight/log/admin.py @@ -59,8 +59,10 @@ def download_link(self, obj): """Get download link for LogFile.""" return format_html( 'Download', - reverse('admin:dashboard_download_log_file', - args=[obj.pk]) + reverse( + 'admin:dashboard_download_log_file', + args=[obj.pk] + ) ) download_link.short_description = 'Download Log File'