Skip to content

Commit a4f1a08

Browse files
redirect logs to files
1 parent 6c2073b commit a4f1a08

File tree

10 files changed

+199
-10
lines changed

10 files changed

+199
-10
lines changed

deployment/.template.env

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,4 +59,11 @@ APP_DOMAIN=localhost
5959
# - reference_dataset
6060
# - machine_info_fetcher
6161
PLUGINS=
62-
# -----------------------------
62+
# -----------------------------
63+
64+
# log rotate max files
65+
LOGROTATE_COPIES=14
66+
# log rotate max size per file
67+
LOGROTATE_SIZE=100M
68+
# log rotate interval check
69+
LOGROTATE_INTERVAL=daily

deployment/docker-compose.override.template.yml

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,13 @@ services:
2020
dbbackups:
2121
volumes:
2222
- ./volumes/backups:/backups
23+
redis:
24+
volumes:
25+
# fix redis permission issue because it's using user redis (1001)
26+
# sudo chown -R 1001:1001 ./volumes/tmp_data/redis
27+
- ./volumes/tmp_data/redis:/tmp
28+
- ./redis/redis.conf:/opt/bitnami/redis/mounted-etc/overrides.conf
29+
2330
django:
2431
build:
2532
context: ../
@@ -30,6 +37,8 @@ services:
3037
- ./volumes/static:/home/web/static
3138
- ./volumes/media:/home/web/media
3239
- ./volumes/backups:/backups
40+
- ./volumes/tmp_data/django:/tmp
41+
- ./volumes/tmp_data:/home/web/logs
3342
worker:
3443
build:
3544
context: ../
@@ -38,6 +47,7 @@ services:
3847
- ../django_project:/home/web/django_project
3948
- ./volumes/static:/home/web/static
4049
- ./volumes/media:/home/web/media
50+
- ./volumes/tmp_data/worker:/tmp
4151
celery_beat:
4252
build:
4353
context: ../
@@ -46,6 +56,7 @@ services:
4656
- ../django_project:/home/web/django_project
4757
- ./volumes/static:/home/web/static
4858
- ./volumes/media:/home/web/media
59+
- ./volumes/tmp_data/celery_beat:/tmp
4960
nginx:
5061
build:
5162
context: ../
@@ -54,6 +65,7 @@ services:
5465
- ./nginx/sites-enabled:/etc/nginx/conf.d
5566
- ./volumes/static:/home/web/static:ro
5667
- ./volumes/media:/home/web/media:ro
68+
- ./volumes/tmp_data/nginx:/tmp
5769
ports:
5870
- "${HTTP_PORT:-80}:8080"
5971
- "${HTTPS_PORT:-443}:443"
@@ -72,6 +84,7 @@ services:
7284
- ./volumes/static:/home/web/static
7385
- ./volumes/media:/home/web/media
7486
- ./volumes/backups:/backups
87+
- ./volumes/tmp_data:/home/web/logs
7588
environment:
7689
# editable in .env
7790
- DATABASE_TEMP_NAME=${DATABASE_TEMP_NAME:-temp}
@@ -113,4 +126,9 @@ services:
113126
links:
114127
- db
115128
- redis
116-
- celery_beat
129+
- celery_beat
130+
131+
logrotate:
132+
volumes:
133+
- ./volumes/tmp_data:/logs
134+
- ./volumes/tmp_data/logrotate:/tmp

deployment/docker-compose.yml

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ volumes:
2020
backups-data:
2121
rabbitmq:
2222
redis-data:
23+
tmp-data:
24+
redis-conf:
25+
tmp-logrotate:
2326

2427
x-common-django:
2528
&default-common-django
@@ -66,6 +69,7 @@ x-common-django:
6669
volumes:
6770
- static-data:/home/web/static
6871
- media-data:/home/web/media
72+
- tmp-data:/tmp
6973
restart: on-failure
7074

7175
services:
@@ -76,6 +80,8 @@ services:
7680
- REDIS_PASSWORD=${REDIS_PASSWORD:-redis_password}
7781
volumes:
7882
- redis-data:/bitnami/redis/data
83+
- tmp-data:/tmp
84+
- redis-conf:/opt/bitnami/redis/mounted-etc/overrides.conf
7985

8086
db:
8187
image: kartoza/postgis:13.0
@@ -121,7 +127,7 @@ services:
121127
worker:
122128
<<: *default-common-django
123129
container_name: "geosight_worker"
124-
command: 'celery -A core worker -l info'
130+
command: 'celery -A core worker -l info --logfile=/tmp/worker.log'
125131
entrypoint: [ ]
126132
links:
127133
- db
@@ -132,7 +138,7 @@ services:
132138
<<: *default-common-django
133139
container_name: "geosight_celery_beat"
134140
entrypoint: [ ]
135-
command: 'celery -A core beat --loglevel=info --scheduler django_celery_beat.schedulers:DatabaseScheduler'
141+
command: 'celery -A core beat --loglevel=info --scheduler django_celery_beat.schedulers:DatabaseScheduler --logfile=/tmp/celery_beat.log'
136142
links:
137143
- db
138144
- redis
@@ -144,6 +150,19 @@ services:
144150
volumes:
145151
- static-data:/home/web/static:ro
146152
- media-data:/home/web/media:ro
153+
- tmp-data:/tmp
147154
ports:
148155
- "${HTTP_PORT:-80}:8080"
149156
restart: on-failure:5
157+
158+
logrotate:
159+
image: blacklabelops/logrotate:1.3
160+
volumes:
161+
- tmp-data:/logs
162+
- tmp-logrotate:/tmp
163+
environment:
164+
- LOGS_DIRECTORIES=/logs
165+
- LOGROTATE_STATUSFILE=/tmp/logrotate.status
166+
- LOGROTATE_COPIES=${LOGROTATE_COPIES:-14}
167+
- LOGROTATE_SIZE=${LOGROTATE_SIZE:-100M}
168+
- LOGROTATE_INTERVAL=${LOGROTATE_INTERVAL:-daily}

deployment/docker/uwsgi.conf

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,11 @@ cheaper = 2
2525
env = DJANGO_SETTINGS_MODULE=core.settings.prod
2626
# disabled so we run in the foreground for docker
2727
# daemonize = /tmp/django.log
28+
logto = /tmp/django.log
2829
# reload-os-env
2930
# uid = 1000
3031
# gid = 1000
3132
memory-report = true
3233
harakiri = 2400
3334
buffer-size = 8192
34-
disable-logging = True
35+
# disable-logging = True

deployment/nginx/nginx.conf

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
user nginx;
22
worker_processes auto;
33

4-
error_log /var/log/nginx/error.log notice;
5-
pid /var/run/nginx.pid;
4+
error_log /tmp/error.log notice;
5+
pid /tmp/nginx.pid;
66

77
events {
88
worker_connections 1024;
@@ -21,7 +21,7 @@ http {
2121
'$status $body_bytes_sent "$http_referer" '
2222
'"$http_user_agent" "$http_x_forwarded_for"';
2323

24-
access_log /var/log/nginx/access.log main if=$logme;
24+
access_log /tmp/access.log main if=$logme;
2525

2626
sendfile on;
2727
#tcp_nopush on;

deployment/redis/redis.conf

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
logfile /tmp/redis.log

django_project/core/settings/project.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,3 +191,9 @@
191191
INSTALLED_APPS = [
192192
app for app in INSTALLED_APPS if app != 'django.contrib.admin'
193193
]
194+
195+
# ----------------------------------------
196+
# Logs Directory
197+
# ----------------------------------------
198+
LOGS_DIRECTORY = os.environ.get(
199+
'LOGS_DIRECTORY', '/home/web/logs')

django_project/geosight/machine_info_fetcher/admin.py

Lines changed: 102 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,108 @@
1414
__date__ = '22/10/2024'
1515
__copyright__ = ('Copyright 2023, Unicef')
1616

17-
from django.contrib import admin
17+
import os
18+
import re
19+
import mimetypes
20+
from datetime import datetime
21+
from django.conf import settings
22+
from django.contrib import admin, messages
23+
from django.urls import re_path, reverse
24+
from django.http import (
25+
HttpResponseRedirect,
26+
HttpResponse,
27+
Http404,
28+
FileResponse
29+
)
30+
from django.utils.html import format_html
31+
32+
from geosight.machine_info_fetcher.models import MachineInfo, LogFile
33+
34+
35+
def list_log_files(parent_dir):
36+
"""Get list of log files from parent_dir."""
37+
log_files = []
38+
for root, dirs, files in os.walk(parent_dir):
39+
for file in files:
40+
if ".log" in file:
41+
file_path = os.path.join(root, file)
42+
file_size = os.path.getsize(file_path)
43+
created_on = datetime.fromtimestamp(
44+
os.path.getctime(file_path)
45+
).strftime('%Y-%m-%d %H:%M:%S')
46+
log_files.append((file_path, file_size, created_on))
47+
return log_files
48+
49+
50+
class LogFileAdmin(admin.ModelAdmin):
51+
"""Admin class for LogFile."""
52+
53+
list_display = (
54+
'filename', 'size', 'created_on', 'download_link')
55+
actions = ['refresh_log_files']
56+
57+
def download_link(self, obj):
58+
return format_html(
59+
'<a href="{}">Download</a>',
60+
reverse('admin:dashboard_download_log_file',
61+
args=[obj.pk])
62+
)
63+
download_link.short_description = 'Download Log File'
64+
65+
def get_urls(self):
66+
urls = super().get_urls()
67+
custom_urls = [
68+
re_path(r'^download-log-file/(?P<pk>\d+)$',
69+
self.download_log_file,
70+
name='dashboard_download_log_file'
71+
),
72+
]
73+
return custom_urls + urls
74+
75+
def download_log_file(self, request, pk):
76+
try:
77+
log_file = LogFile.objects.get(pk=pk)
78+
file_path = log_file.path
79+
file_handle = open(file_path, 'rb')
80+
81+
# Use FileResponse for efficient streaming
82+
response = FileResponse(
83+
file_handle,
84+
content_type=mimetypes.guess_type(file_path)[0]
85+
)
86+
response['Content-Disposition'] = (
87+
f'attachment; filename="{log_file.filename()}"'
88+
)
89+
return response
90+
except LogFile.DoesNotExist:
91+
raise Http404("Log file not found")
92+
except Exception as e:
93+
return HttpResponse(f"Error: {e}", status=500)
94+
95+
def refresh_log_files(self, request, queryset):
96+
"""Refresh log file list."""
97+
LogFile.objects.all().delete()
98+
99+
# Insert new log files
100+
new_log_files = list_log_files(settings.LOGS_DIRECTORY)
101+
log_entries = [
102+
LogFile(path=path, size=size, created_on=created_on)
103+
for path, size, created_on in new_log_files
104+
]
105+
LogFile.objects.bulk_create(log_entries)
106+
107+
# Send a success message
108+
self.message_user(
109+
request,
110+
"Log files have been refreshed.",
111+
level=messages.SUCCESS
112+
)
113+
114+
# Redirect back to the log file list page
115+
return HttpResponseRedirect(request.get_full_path())
116+
117+
refresh_log_files.short_description = "Refresh log files"
18118

19-
from geosight.machine_info_fetcher.models import MachineInfo
20119

21120
admin.site.register(MachineInfo, admin.ModelAdmin)
121+
admin.site.register(LogFile, LogFileAdmin)
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Generated by Django 3.2.16 on 2024-11-12 16:15
2+
3+
from django.db import migrations, models
4+
5+
6+
class Migration(migrations.Migration):
7+
8+
dependencies = [
9+
('geosight_machine_info_fetcher', '0001_initial'),
10+
]
11+
12+
operations = [
13+
migrations.CreateModel(
14+
name='LogFile',
15+
fields=[
16+
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
17+
('path', models.CharField(max_length=500, unique=True)),
18+
('size', models.PositiveBigIntegerField()),
19+
('created_on', models.DateTimeField()),
20+
],
21+
),
22+
]

django_project/geosight/machine_info_fetcher/models.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,18 @@ class MachineInfo(models.Model):
3838

3939
def __str__(self):
4040
return f'{self.date_time}'
41+
42+
43+
class LogFile(models.Model):
44+
"""Class that represent log file."""
45+
46+
path = models.CharField(max_length=500, unique=True)
47+
size = models.PositiveBigIntegerField()
48+
created_on = models.DateTimeField()
49+
50+
def filename(self):
51+
"""Extracts the filename from the path."""
52+
return self.path.split('/')[-1]
53+
54+
def __str__(self):
55+
return self.filename()

0 commit comments

Comments
 (0)