Merge pull request #305 from sparcs-kaist/develop

2022/02/15 v1.2.9 update

apps/core/management/scripts/portal_crawler.py

@@ -1,4 +1,6 @@
+import hashlib
 import re
+import boto3
 import uuid
 from datetime import datetime
 
@@ -14,7 +16,7 @@
 
 from apps.core.models import Article
 from apps.user.models import UserProfile
-from ara.settings import PORTAL_ID, PORTAL_PASSWORD, PORTAL_2FA_KEY
+from ara.settings import PORTAL_ID, PORTAL_PASSWORD, PORTAL_2FA_KEY, AWS_S3_BUCKET_NAME
 
 LOGIN_INFO_SSO2 = {
     'userid': PORTAL_ID,
@@ -120,6 +122,30 @@ def _enable_hyperlink(s):
 
         return new_string
 
+    def _get_new_url_and_save_to_s3(url, session):
+        if '.' in url.split('/')[-1]: # not a portal image
+            return url
+        enc = hashlib.md5()
+        enc.update(url.encode())
+        hash = enc.hexdigest()[:20]
+        filename = f'files/portal_image_{hash}.{url.split("_")[-1]}'
+
+        r = session.get(url, stream=True)
+        if r.status_code == 200:
+            s3 = boto3.client('s3')
+            s3.upload_fileobj(r.raw, Bucket = AWS_S3_BUCKET_NAME, Key = filename)
+
+        return f'https://{AWS_S3_BUCKET_NAME}.s3.amazonaws.com/{filename}'
+
+    def _save_portal_image(html, session):
+        soup = bs(html, 'lxml')
+        for child in soup.find_all('img', {}):
+            old_url = child.attrs.get('src')
+            new_url = _get_new_url_and_save_to_s3(old_url, session)
+            child['src'] = new_url
+
+        return str(soup)
+
     article_req = session.get(url)
     soup = bs(article_req.text, 'lxml')
 
@@ -150,6 +176,7 @@ def _enable_hyperlink(s):
             html = tr.find('td').prettify()
             break
 
+    html = _save_portal_image(html, session)
     html = _enable_hyperlink(html)
 
     if html is None:

apps/core/models/article.py

@@ -21,7 +21,7 @@
 from .comment import Comment
 
 
-class ArticleHiddenReason(Enum):
+class ArticleHiddenReason(str, Enum):
     ADULT_CONTENT = 'ADULT_CONTENT'
     SOCIAL_CONTENT = 'SOCIAL_CONTENT'
     REPORTED_CONTENT = 'REPORTED_CONTENT'

apps/core/models/notification.py

@@ -60,27 +60,27 @@ def data(self) -> dict:
     def notify_commented(cls, comment):
         from apps.core.models import NotificationReadLog
 
-        def notify_article_commented(_article, _comment):
+        def notify_article_commented(_parent_article, _comment):
             NotificationReadLog.objects.create(
-                read_by=_article.created_by,
+                read_by=_parent_article.created_by,
                 notification=cls.objects.create(
                     type='article_commented',
                     title='회원님의 게시물에 새로운 댓글이 작성되었습니다.',
                     content=_comment.content[:32],
-                    related_article=_article,
-                    related_comment=_comment,
+                    related_article=_parent_article,
+                    related_comment=None,
                 ),
             )
 
-        def notify_comment_commented(_article, _comment):
+        def notify_comment_commented(_parent_article, _comment):
             NotificationReadLog.objects.create(
                 read_by=_comment.parent_comment.created_by,
                 notification=cls.objects.create(
                     type='comment_commented',
                     title='회원님의 댓글에 새로운 댓글이 작성되었습니다.',
                     content=_comment.content[:32],
-                    related_article=_article,
-                    related_comment=_comment,
+                    related_article=_parent_article,
+                    related_comment=_comment.parent_comment,
                 ),
             )
 

apps/core/serializers/article.py

@@ -1,11 +1,10 @@
 import typing
 
-from django.db import models
+from enum import Enum
 from django.utils.translation import gettext
 from rest_framework import serializers
-
 from apps.core.documents import ArticleDocument
-from apps.core.models import Article, ArticleReadLog, Board, Block, Scrap, ArticleHiddenReason
+from apps.core.models import Article, Board, Block, Scrap, ArticleHiddenReason
 from apps.core.serializers.board import BoardSerializer
 from apps.core.serializers.mixins.hidden import HiddenSerializerMixin, HiddenSerializerFieldMixin
 from apps.core.serializers.topic import TopicSerializer
@@ -315,6 +314,13 @@ def get_attachments(self, obj): # -> typing.Optional[list]:
     )
 
 
+class ArticleAttachmentType(Enum):
+    NONE = 'NONE'
+    IMAGE = 'IMAGE'
+    NON_IMAGE = 'NON_IMAGE'
+    BOTH = 'BOTH'
+
+
 class ArticleListActionSerializer(HiddenSerializerFieldMixin, BaseArticleSerializer):
     parent_topic = TopicSerializer(
         read_only=True,
@@ -332,6 +338,29 @@ class ArticleListActionSerializer(HiddenSerializerFieldMixin, BaseArticleSeriali
         read_only=True,
     )
 
+    attachment_type = serializers.SerializerMethodField(
+        read_only=True,
+    )
+
+    def get_attachment_type(self, obj) -> str:
+        if not self.visible_verdict(obj):
+            return ArticleAttachmentType.NONE.value
+
+        has_image = False
+        has_non_image = False
+        for att in obj.attachments.all():
+            if att.mimetype[:5] == 'image':
+                has_image = True
+            else:
+                has_non_image = True
+        if has_image and has_non_image:
+            return ArticleAttachmentType.BOTH.value
+        if has_image:
+            return ArticleAttachmentType.IMAGE.value
+        if has_non_image:
+            return ArticleAttachmentType.NON_IMAGE.value
+        return ArticleAttachmentType.NONE.value
+
 
 class BestArticleListActionSerializer(HiddenSerializerFieldMixin, BaseArticleSerializer):
     title = serializers.SerializerMethodField(

apps/core/views/viewsets/article.py

@@ -1,6 +1,7 @@
 import time
 
 from django.db import models
+from django.http import Http404
 from django.utils.translation import gettext
 from rest_framework import status, viewsets, response, decorators, serializers, permissions
 from rest_framework.response import Response
@@ -169,7 +170,14 @@ def perform_destroy(self, instance):
         return super().perform_destroy(instance)
 
     def retrieve(self, request, *args, **kwargs):
-        article = self.get_object()
+        try:
+            article = self.get_object()
+        except Http404 as e:
+            if Article.objects.queryset_with_deleted.filter(id=kwargs['pk']).exists():
+                return response.Response(status=status.HTTP_410_GONE)
+            else:
+                raise e
+
         override_hidden = 'override_hidden' in self.request.query_params
 
         ArticleReadLog.objects.create(

tests/test_articles.py

@@ -655,6 +655,21 @@ def test_modify_report_hidden_article(self):
 
         assert res.status_code == 403
 
+    def test_get_deleted_article(self):
+        target_article = self._create_deleted_article()
+
+        res = self.http_request(self.user2, 'get', f'articles/{target_article.id}')
+        assert res.status_code == 410
+
+    def test_exclude_deleted_article_from_list(self):
+        target_article = self._create_deleted_article()
+
+        res = self.http_request(self.user2, 'get', f'articles').data
+
+        # user가 글 목록을 가져올 때, 삭제된 글이 목록에 없는 것 확인
+        for post in res.get('results'):
+            assert post.get('id') != target_article.id
+
     def test_delete_already_deleted_article(self):
         target_article = self._create_deleted_article()
         res = self.http_request(self.user, 'delete', f'articles/{target_article.id}')

tests/test_notification.py

@@ -37,14 +37,14 @@ def set_comment(request):
     )
 
 
-@pytest.mark.usefixtures('set_user_client', 'set_user_client2', 'set_board', 'set_articles')
+@pytest.mark.usefixtures('set_user_client', 'set_user_client2', 'set_board', 'set_articles', 'set_comment')
 class TestNotification(TestCase, RequestSetting):
-    @pytest.mark.usefixtures('set_comment')
     def test_notification_article_commented(self):
         notifications = self.http_request(self.user, 'get', 'notifications')
 
         # user에게 알림: user의 글에 user2가 댓글을 달아서
         assert notifications.status_code == 200
+        assert notifications.data.get('results')[0].get('related_article')['id'] == self.article.id
         assert Notification.objects.all().count() == 1
 
         assert notifications.data.get('num_items') == 1
@@ -60,7 +60,8 @@ def test_notification_comment_commented(self):
 
         # user2에게 알림: user2의 댓글에 user가 대댓글을 달아서
         assert notifications.status_code == 200
-        assert Notification.objects.filter(related_comment=cc).count() == 1
+        assert notifications.data.get('results')[0].get('related_comment')['id'] == self.comment.id
+        assert Notification.objects.filter(related_comment=self.comment).count() == 1
 
         assert notifications.data.get('num_items') == 1