Merge pull request #56 from SADiLaR/feature/pdf-package-install

saving PDF text into database

.env.example

@@ -9,3 +9,4 @@ LOGGING_FILE=debug.log
 LOGGING_HANDLERS_LEVEL=INFO
 LOGGING_LOGGERS_LEVEL=INFO
 LOGGING_LOGGERS_DJANGO_LEVEL=INFO
+TESTING_DIR=/app/general/tests/files/

.env.testing

@@ -9,3 +9,4 @@ LOGGING_FILE=debug.log
 LOGGING_HANDLERS_LEVEL=INFO
 LOGGING_LOGGERS_LEVEL=INFO
 LOGGING_LOGGERS_DJANGO_LEVEL=INFO
+TESTING_DIR=/home/runner/work/term_platform/term_platform/app/general/tests/files/

app/general/admin.py

@@ -1,6 +1,8 @@
 import magic
 from django.contrib import admin
 from django.forms import HiddenInput, ModelForm
+from pypdf import PdfReader
+from pypdf.errors import PdfStreamError
 from simple_history.admin import SimpleHistoryAdmin
 
 from .models import DocumentFile, Institution, Language, Project, Subject
@@ -14,6 +16,8 @@ class Meta:
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
+        self.fields["document_data"].widget = HiddenInput()
+
         # If the instance has a mime_type, the field should be disabled
         if not self.instance.mime_type:
             self.fields["mime_type"].widget = HiddenInput()
@@ -30,6 +34,9 @@ def clean(self):
             if file_type != "application/pdf":
                 self.add_error("uploaded_file", "Only PDF files are allowed.")
 
+            # Extract text from PDF file
+            cleaned_data["document_data"] = self.pdf_to_text(uploaded_file)
+
             cleaned_data["mime_type"] = file_type
 
             uploaded_file.seek(0)  # Reset file pointer after read
@@ -45,6 +52,25 @@ def clean(self):
 
         return cleaned_data
 
+    def pdf_to_text(self, uploaded_file):
+        if uploaded_file:
+            text_list = []
+            # Read the PDF file and extract text
+            try:
+                reader = PdfReader(uploaded_file)
+                for page in reader.pages:
+                    text_list.append(page.extract_text())
+
+                get_pdf_text = " ".join(text_list)
+
+                return str(get_pdf_text)
+
+            except PdfStreamError:
+                return self.add_error(
+                    "uploaded_file", "The uploaded PDF file is corrupted or not fully downloaded."
+                )
+        return None
+
 
 class DocumentFileAdmin(SimpleHistoryAdmin):
     ordering = ["title"]

app/general/migrations/0006_documentfile_document_data.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.2 on 2024-05-23 12:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('general', '0005_alter_institution_logo_alter_project_logo'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='documentfile',
+            name='document_data',
+            field=models.TextField(blank=True),
+        ),
+    ]

app/general/models.py

@@ -107,12 +107,13 @@ class DocumentFile(models.Model):
         max_length=200, blank=True, help_text="This input will auto-populate."
     )
     document_type = models.CharField(max_length=200, choices=document_type_choices)
+    document_data = models.TextField(blank=True)
     institution = models.ForeignKey("Institution", on_delete=models.CASCADE)
     subjects = models.ManyToManyField("Subject", blank=True)
     languages = models.ManyToManyField("Language", blank=True)
 
     # added simple historical records to the model
-    history = HistoricalRecords()
+    history = HistoricalRecords(excluded_fields=["document_data"])
 
     def __str__(self):
         return self.title

app/general/tests/test_document_admin_file.py

@@ -1,3 +1,4 @@
+import os
 import unittest
 
 from django.core.files.uploadedfile import SimpleUploadedFile
@@ -12,7 +13,11 @@ def __init__(self, methodName: str = "runTest"):
         self.form = None
 
     def setUp(self):
-        pdf_file = b"%PDF-1.1 0 obj<</Pages 2 0 R>>endobj2 0 obj<</Kids[3 0 R]/Count 1>>endobj3 0 obj<</Parent 2 0 R>>endobjtrailer <</Root 1 0 R>>"
+        test_dir = os.getenv("TESTING_DIR", "/app/general/tests/files")
+        test_file = test_dir + "/Lorem.pdf"
+
+        with open(test_file, "rb") as f:
+            pdf_file = f.read()
 
         self.file_mock = SimpleUploadedFile("test.pdf", pdf_file, content_type="application/pdf")
 
@@ -43,11 +48,13 @@ def test_clean_without_file(self):
             "institution": Institution.objects.create(name="Test Institution 2"),
             "url": "www.example.com",
             "uploaded_file": "",
+            "document_data": "",
         }
 
         form = DocumentFileForm(tests_form)
         self.assertTrue(form.is_valid())
 
+    #
     def test_clean_without_url(self):
         tests_form = {
             "title": "Test",
@@ -57,6 +64,7 @@ def test_clean_without_url(self):
             "institution": Institution.objects.create(name="Test Institution 3"),
             "url": "",
             "uploaded_file": self.file_mock,
+            "document_data": "",
         }
 
         form = DocumentFileForm(tests_form, files={"uploaded_file": self.file_mock})

docker-compose.yml

@@ -1,6 +1,6 @@
 # This is is meant for local development, but should give an idea of what you
 # can consider in production.
-version: '3'
+version: '3.8'
 
 services:
   db:
@@ -37,3 +37,4 @@ services:
       - DB_NAME=term_db  # see POSTGRES_DB above
       - DB_USER=sadilar  # see POSTGRES_USER above
       - DB_PASSWORD=sadilar  # see POSTGRES_PASSWORD above
+      - TESTING_DIR=/app/general/tests/files/

requirements.txt

@@ -6,3 +6,4 @@ psycopg2-binary
 whitenoise
 pillow
 python-magic
+pypdf