diff --git a/Pipfile b/Pipfile index 62e28480..1a72aa4e 100644 --- a/Pipfile +++ b/Pipfile @@ -27,6 +27,7 @@ django-modeltranslation = {editable = true, git = "https://github.com/lifenautjo bandit = "*" pyyaml = "*" python-magic = "*" +django-imagekit = "*" [pipenv] allow_prereleases = true diff --git a/Pipfile.lock b/Pipfile.lock index 8e693f79..ee06cdc8 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "0ac00cccccca5c80c660c7b2ec3890372d55acb152cabdb457962d9917450fe7" + "sha256": "2c1623af3262f227ea1bfd8bf7ed294771464e0ae81792ef11c0f02f959b3337" }, "pipfile-spec": 6, "requires": {}, @@ -24,18 +24,18 @@ }, "boto3": { "hashes": [ - "sha256:1ed36e390a64d9f45fbcc71f835a43b2fef437cc89d6fd9306c6396b10b8ec29", - "sha256:db5704a505d8af01cdbc57682797e8b1784a31a37d24a82f4855c270cd5f8b2f" + "sha256:428c6d535f373a7203ed7ec687bb826b88e62de7befc3816e0aacd305f7572a2", + "sha256:4a693170c79b4275d8bb6884e9930c82e6bb4e303597ca545f288333db77c6a7" ], "index": "pypi", - "version": "==1.9.74" + "version": "==1.9.91" }, "botocore": { "hashes": [ - "sha256:0d21a6a163033792c759d8bc03a9267cd00813b3c2136cc626a0a2d0ed7f7cfb", - "sha256:1af5c692b8af8c6f955af8e2e4daad3186eab382ead24b517006462b0f71cc13" + "sha256:4ca7da7128915d7ac149e12f8a3efeb4e590793189cabe0bcd3c3eee5b84f656", + "sha256:bd788c6ebae55db17d9cc125fa3817fddb20d3fc8bd15791995d82c466238a3b" ], - "version": "==1.12.74" + "version": "==1.12.91" }, "certifi": { "hashes": [ @@ -88,11 +88,11 @@ }, "django": { "hashes": [ - "sha256:a32c22af23634e1d11425574dce756098e015a165be02e4690179889b207c7a8", - "sha256:d6393918da830530a9516bbbcbf7f1214c3d733738779f06b0f649f49cc698c3" + "sha256:146aa583364553c9ecbed55613a6c27bc23226048f86ed183350c80df3f2a844", + "sha256:737bd3d5f70cb8bde3c660e69b077a0221b47caf499d4d3759fb086376002d4a" ], "index": "pypi", - "version": "==2.1.5" + "version": "==2.2a1" }, "django-amazon-ses": { "hashes": [ @@ -102,6 +102,21 @@ "index": "pypi", "version": "==2.0.0" }, + "django-appconf": { + "hashes": [ + "sha256:6a4d9aea683b4c224d97ab8ee11ad2d29a37072c0c6c509896dd9857466fb261", + "sha256:ddab987d14b26731352c01ee69c090a4ebfc9141ed223bef039d79587f22acd9" + ], + "version": "==1.0.2" + }, + "django-imagekit": { + "hashes": [ + "sha256:304c3379f6a5cac387e47ace11195a603ad3cb01e3e951b45489824d25b00359", + "sha256:6ec0afb77cdf52cd453c9fc2c10ef350d111edfd6ce53c5977aa8a0e22cee00c" + ], + "index": "pypi", + "version": "==4.0.2" + }, "django-media-fixtures": { "hashes": [ "sha256:735a95fbe40b4ce50951451157f9bb27f7f836fbee9bf59b21550acc11cf5295" @@ -132,11 +147,11 @@ }, "djangorestframework": { "hashes": [ - "sha256:607865b0bb1598b153793892101d881466bd5a991de12bd6229abb18b1c86136", - "sha256:63f76cbe1e7d12b94c357d7e54401103b2e52aef0f7c1650d6c820ad708776e5" + "sha256:79c6efbb2514bc50cf25906d7c0a5cfead714c7af667ff4bd110312cd380ae66", + "sha256:a4138613b67e3a223be6c97f53b13d759c5b90d2b433bad670b8ebf95402075f" ], "index": "pypi", - "version": "==3.9.0" + "version": "==3.9.1" }, "docutils": { "hashes": [ @@ -185,10 +200,12 @@ }, "mysqlclient": { "hashes": [ - "sha256:37496c1181805e4001164a18a4383962901b03c03e148cd165a2619515ccc988" + "sha256:041c79d474cd0b4980f1175f1ff24d2796d6e1f1e400583b40d21ed0d5a4f279", + "sha256:b95edaa41d6cc47deecabcdcbb5ab437ad9ae6d8955f5cf10d1847b37e66ef5e", + "sha256:cd07e321f1f692ecd67e8291ffbacd61c7b42a6cedc84d40971fbddbbce9b79e" ], "index": "pypi", - "version": "==1.4.0rc3" + "version": "==1.4.2" }, "nose": { "hashes": [ @@ -208,46 +225,52 @@ }, "pbr": { "hashes": [ - "sha256:f59d71442f9ece3dffc17bc36575768e1ee9967756e6b6535f0ee1f0054c3d68", - "sha256:f6d5b23f226a2ba58e14e49aa3b1bfaf814d0199144b95d78458212444de1387" + "sha256:a7953f66e1f82e4b061f43096a4bcc058f7d3d41de9b94ac871770e8bdd831a2", + "sha256:d717573351cfe09f49df61906cd272abaa759b3e91744396b804965ff7bff38b" ], - "version": "==5.1.1" + "version": "==5.1.2" + }, + "pilkit": { + "hashes": [ + "sha256:ddb30c2f0198a147e56b151476c3bb9fe045fbfd5b0a0fa2a3148dba62d1559f" + ], + "version": "==2.0" }, "pillow": { "hashes": [ - "sha256:0cd42fe2d99ec6ce23aaf00947a7b7956ad2ed4b1695fd37545c3b8eae06d95a", - "sha256:137bed8972089d65da63fb79b4949b0f2b99e9a58f1b494e82be43ba8b0f4226", - "sha256:14eb2b2e4f2a14f5c89fd0edf55c5af0bf1a40fdf3838d81867f26f131cd557d", - "sha256:1fc43ce8c4fa3754222cd6831d599ad17ca2fc9868d2fb52f4e5362dfbfaf379", - "sha256:26dfeee23a86dad6277a63d18f61f53b957cb2cd3506dbbd74b88ba2fa65b3b1", - "sha256:2e0e582942e025cc58f669499a8e0bffde5bcc8d42b65729f294c1dac54e4672", - "sha256:3bb8dd3ce101dd8b0b37eaae924a5bb93abb6ffdd034bf68a066a808e11768ab", - "sha256:3f07da3874f0b085421f1d4f979785131aa9d497501d8610d82f7378b33858f8", - "sha256:429b2b5ae5f57f8fd9ec2e012c1e7b342ff10f1a8977dc291976b9a3b4c096e1", - "sha256:4a000fdd89d77b6b675de27e1ab91c6fba517c08f19ee83e6716b78930634e04", - "sha256:4ccbe7cce6156391a3ecf447c79a7d4a1a0ecd3de79bdec9ca5e4f7242a306d1", - "sha256:4d08034196db41acb7392e4fccfc0448e7a87192c41d3011ad4093eac2c31ffd", - "sha256:6b202b1cb524bc76ed52a7eb0314f4b0a0497c7cceb9a93539b5a25800e1f2b6", - "sha256:8563b56fa7c34f1606848c2143ea67d27cf225b9726a1b041c3d27cf85e46edc", - "sha256:86d7421e8803d7bae2e594765c378a867b629d46b32fbfe5ed9fd95b30989feb", - "sha256:8d4bddedcb4ab99131d9705a75720efc48b3d006122dae1a4cc329496ac47c9a", - "sha256:a4929c6de9590635c34533609402c9da12b22bfc2feb8c0c4f38c39bab48a9ad", - "sha256:b0736e21798448cee3e663c0df7a6dfa83d805b3f3a45e67f7457a2f019e5fca", - "sha256:b669acba91d47395de84c9ca52a7ad393b487e5ae2e20b9b2790b22a57d479fa", - "sha256:bba993443921f2d077195b425a3283357f52b07807d53704610c1249d20b183a", - "sha256:bdf706a93d00547c9443b2654ae424fd54d5dece4bc4333e7035740aeb7a7cea", - "sha256:c5aa93e55175b9cde95279ccd03c93d218976b376480222d37be41d2c9c54510", - "sha256:cc11fd997d8ad71bb0412e983b711e49639c2ddba9b9dce04d4bdab575fe5f84", - "sha256:d584f1c33995c3dc16a35e30ef43e0881fa0d085f0fef29cebf154ffb5643363", - "sha256:d88f54bdefb7ddccb68efdd710d689aa6a09b875cc3e44b7e81ef54e0751e3a7", - "sha256:de0d323072be72fa4d74f4e013cd594e3f8ee03b2e0eac5876a3249fa076ef7b", - "sha256:f139c963c6679d236b2c45369524338eabd36a853fe23abd39ba246ab0a75aec", - "sha256:f41c0bf667c4c1c30b873eaa8d6bb894f6d721b3e38e9c993bddd1263c02fb1f", - "sha256:fbd0ea468b4ec04270533bf5206f1cd57746fcf226520bb133318fa276de2644", - "sha256:fe2d2850521c467c915ff0a6e27dc64c3c04c2f66612e0072672bd1bd4854b61" + "sha256:051de330a06c99d6f84bcf582960487835bcae3fc99365185dc2d4f65a390c0e", + "sha256:0ae5289948c5e0a16574750021bd8be921c27d4e3527800dc9c2c1d2abc81bf7", + "sha256:0b1efce03619cdbf8bcc61cfae81fcda59249a469f31c6735ea59badd4a6f58a", + "sha256:163136e09bd1d6c6c6026b0a662976e86c58b932b964f255ff384ecc8c3cefa3", + "sha256:18e912a6ccddf28defa196bd2021fe33600cbe5da1aa2f2e2c6df15f720b73d1", + "sha256:24ec3dea52339a610d34401d2d53d0fb3c7fd08e34b20c95d2ad3973193591f1", + "sha256:267f8e4c0a1d7e36e97c6a604f5b03ef58e2b81c1becb4fccecddcb37e063cc7", + "sha256:3273a28734175feebbe4d0a4cde04d4ed20f620b9b506d26f44379d3c72304e1", + "sha256:4c678e23006798fc8b6f4cef2eaad267d53ff4c1779bd1af8725cc11b72a63f3", + "sha256:4d4bc2e6bb6861103ea4655d6b6f67af8e5336e7216e20fff3e18ffa95d7a055", + "sha256:505738076350a337c1740a31646e1de09a164c62c07db3b996abdc0f9d2e50cf", + "sha256:5233664eadfa342c639b9b9977190d64ad7aca4edc51a966394d7e08e7f38a9f", + "sha256:5d95cb9f6cced2628f3e4de7e795e98b2659dfcc7176ab4a01a8b48c2c2f488f", + "sha256:7eda4c737637af74bac4b23aa82ea6fbb19002552be85f0b89bc27e3a762d239", + "sha256:801ddaa69659b36abf4694fed5aa9f61d1ecf2daaa6c92541bbbbb775d97b9fe", + "sha256:825aa6d222ce2c2b90d34a0ea31914e141a85edefc07e17342f1d2fdf121c07c", + "sha256:9c215442ff8249d41ff58700e91ef61d74f47dfd431a50253e1a1ca9436b0697", + "sha256:a3d90022f2202bbb14da991f26ca7a30b7e4c62bf0f8bf9825603b22d7e87494", + "sha256:a631fd36a9823638fe700d9225f9698fb59d049c942d322d4c09544dc2115356", + "sha256:a6523a23a205be0fe664b6b8747a5c86d55da960d9586db039eec9f5c269c0e6", + "sha256:a756ecf9f4b9b3ed49a680a649af45a8767ad038de39e6c030919c2f443eb000", + "sha256:b117287a5bdc81f1bac891187275ec7e829e961b8032c9e5ff38b70fd036c78f", + "sha256:ba04f57d1715ca5ff74bb7f8a818bf929a204b3b3c2c2826d1e1cc3b1c13398c", + "sha256:cd878195166723f30865e05d87cbaf9421614501a4bd48792c5ed28f90fd36ca", + "sha256:cee815cc62d136e96cf76771b9d3eb58e0777ec18ea50de5cfcede8a7c429aa8", + "sha256:d1722b7aa4b40cf93ac3c80d3edd48bf93b9208241d166a14ad8e7a20ee1d4f3", + "sha256:d7c1c06246b05529f9984435fc4fa5a545ea26606e7f450bdbe00c153f5aeaad", + "sha256:e9c8066249c040efdda84793a2a669076f92a301ceabe69202446abb4c5c5ef9", + "sha256:f227d7e574d050ff3996049e086e1f18c7bd2d067ef24131e50a1d3fe5831fbc", + "sha256:fc9a12aad714af36cf3ad0275a96a733526571e52710319855628f476dcb144e" ], "index": "pypi", - "version": "==5.4.0" + "version": "==5.4.1" }, "pinocchio": { "hashes": [ @@ -258,11 +281,11 @@ }, "python-dateutil": { "hashes": [ - "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93", - "sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02" + "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", + "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" ], "markers": "python_version >= '2.7'", - "version": "==2.7.5" + "version": "==2.8.0" }, "python-dotenv": { "hashes": [ @@ -282,10 +305,10 @@ }, "pytz": { "hashes": [ - "sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca", - "sha256:8e0f8568c118d3077b46be7d654cc8167fa916092e28320cde048e54bfc9f1e6" + "sha256:32b0891edff07e28efe91284ed9c31e123d84bea3fd98e1f72be2508f43ef8d9", + "sha256:d5f05e487007e29e03409f9398d074e158d920d36eb82eaf66fb1136b0c5374c" ], - "version": "==2018.7" + "version": "==2018.9" }, "pyyaml": { "hashes": [ @@ -300,18 +323,18 @@ }, "s3transfer": { "hashes": [ - "sha256:90dc18e028989c609146e241ea153250be451e05ecc0c2832565231dacdf59c1", - "sha256:c7a9ec356982d5e9ab2d4b46391a7d6a950e2b04c472419f5fdec70cc0ada72f" + "sha256:7b9ad3213bff7d357f888e0fab5101b56fa1a0548ee77d121c3a3dbfbef4cb2e", + "sha256:f23d5cb7d862b104401d9021fc82e5fa0e0cf57b7660a1331425aab0c691d021" ], - "version": "==0.1.13" + "version": "==0.2.0" }, "sentry-sdk": { "hashes": [ - "sha256:2d12914aa71845670aea9555f63091bf55c953ed07a5e64bffa5a149f0a2b8dd", - "sha256:dc775d5d6672cf555d721c3cc9329cc9caed6369ef9aa1b87286017d4ad49614" + "sha256:131e3b9ac11dffd86fe4f1f5d388d3dab372fc9e30d6611d1fc87096a1d67359", + "sha256:e925a2363178c211ad787f507cedda12ee5b0aadf5ac390950140393636a80bb" ], "index": "pypi", - "version": "==0.6.6" + "version": "==0.7.2" }, "six": { "hashes": [ @@ -327,6 +350,13 @@ ], "version": "==2.0.5" }, + "sqlparse": { + "hashes": [ + "sha256:ce028444cfab83be538752a2ffdb56bc417b7784ff35bb9a3062413717807dec", + "sha256:d9cf190f51cbb26da0412247dfe4fb5f4098edb73db84e02f9fc21fdca31fed4" + ], + "version": "==0.2.4" + }, "stevedore": { "hashes": [ "sha256:b92bc7add1a53fb76c634a178978d113330aaf2006f9498d9e2414b31fbfc104", diff --git a/openbook/settings.py b/openbook/settings.py index 9a48df54..c2788b14 100644 --- a/openbook/settings.py +++ b/openbook/settings.py @@ -30,12 +30,19 @@ 'console': { # exact format is not important, this is the minimum information 'format': '%(asctime)s %(name)-12s %(levelname)-8s %(message)s', - } + }, + 'security': { + 'format': '%(asctime)s %(name)-12s %(levelname)-8s %(message)s', + }, }, 'handlers': { 'console': { 'class': 'logging.StreamHandler', 'formatter': 'console', + }, + 'file': { + 'class': 'logging.StreamHandler', + 'formatter': 'console', } }, 'loggers': { @@ -44,6 +51,10 @@ 'level': 'INFO', 'handlers': ['console'], }, + 'security': { + 'level': 'INFO', + 'handlers': ['console'], + }, }, }) @@ -103,6 +114,7 @@ 'django.contrib.staticfiles', 'rest_framework', 'rest_framework.authtoken', + 'imagekit', 'django_nose', 'storages', 'django_media_fixtures', diff --git a/openbook/urls.py b/openbook/urls.py index 90e368c9..21121c84 100644 --- a/openbook/urls.py +++ b/openbook/urls.py @@ -30,7 +30,7 @@ from openbook_posts.views.post.views import PostComments, PostCommentItem, PostItem, PostReactions, PostReactionItem, \ PostReactionsEmojiCount, PostReactionEmojiGroups from openbook_posts.views.posts.views import Posts, TrendingPosts -from openbook_importer.views import ImportItem +from openbook_importer.views import ImportItem, ImportedItem, ImportedItems auth_patterns = [ path('register/', Register.as_view(), name='register-user'), @@ -88,7 +88,9 @@ ] importer_patterns = [ - path('upload/', ImportItem.as_view(), name='uploads') + path('upload/', ImportItem.as_view(), name='uploads'), + path('archives/', ImportedItems.as_view(), name='imported-archives'), + path('archive/', ImportedItem.as_view(), name='imported-archive'), ] api_patterns = [ diff --git a/openbook_auth/migrations/0013_auto_20190209_1631.py b/openbook_auth/migrations/0013_auto_20190209_1631.py new file mode 100644 index 00000000..85454067 --- /dev/null +++ b/openbook_auth/migrations/0013_auto_20190209_1631.py @@ -0,0 +1,19 @@ +# Generated by Django 2.1.5 on 2019-02-09 15:31 + +from django.db import migrations +import imagekit.models.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('openbook_auth', '0012_merge_20181207_1647'), + ] + + operations = [ + migrations.AlterField( + model_name='userprofile', + name='avatar', + field=imagekit.models.fields.ProcessedImageField(null=True, upload_to='', verbose_name='avatar'), + ), + ] diff --git a/openbook_auth/models.py b/openbook_auth/models.py index d200c5ee..305f3812 100644 --- a/openbook_auth/models.py +++ b/openbook_auth/models.py @@ -11,6 +11,9 @@ from rest_framework.exceptions import ValidationError from django.db.models import Q +from imagekit.processors import ResizeToFill +from imagekit.models import ProcessedImageField + from openbook.settings import USERNAME_MAX_LENGTH from openbook_auth.exceptions import EmailVerificationTokenInvalid from openbook_common.utils.model_loaders import get_connection_model, get_circle_model, get_follow_model, \ @@ -335,6 +338,9 @@ def has_reacted_to_post_with_id(self, post_id, emoji_id=None): def has_commented_post_with_id(self, post_id): return self.posts_comments.filter(post_id=post_id).count() > 0 + def has_archive_with_id(self, archive_id): + return self.imports.filter(uuid=archive_id).exists() + def get_lists_for_follow_for_user_with_id(self, user_id): self._check_is_following_user_with_id(user_id) follow = self.get_follow_for_user_with_id(user_id) @@ -880,6 +886,10 @@ def get_connection_for_user_with_id(self, user_id): def get_follow_for_user_with_id(self, user_id): return self.follows.get(followed_user_id=user_id) + def delete_archive_with_id(self, archive_id): + archive = self._check_can_delete_archive_with_id(archive_id) + return self.imports.filter(uuid=archive_id).delete() + def _make_get_post_with_id_query_for_user(self, user, post_id): posts_query = self._make_get_posts_query_for_user(user) posts_query.add(Q(id=post_id), Q.AND) @@ -1218,6 +1228,12 @@ def _check_list_name_not_taken(self, list_name): _('You already have a list with that name.'), ) + def _check_can_delete_archive_with_id(self, archive_id): + if not self.has_archive_with_id(archive_id): + raise ValidationError( + _('Can\'t delete an archive that does not belong to you.'), + ) + @receiver(post_save, sender=settings.AUTH_USER_MODEL) def create_auth_token(sender, instance=None, created=False, **kwargs): @@ -1244,7 +1260,8 @@ class UserProfile(models.Model): location = models.CharField(_('location'), max_length=settings.PROFILE_LOCATION_MAX_LENGTH, blank=False, null=True) user = models.OneToOneField(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name='profile') birth_date = models.DateField(_('birth date'), null=False, blank=False) - avatar = models.ImageField(_('avatar'), blank=False, null=True) + avatar = ProcessedImageField(verbose_name=_('avatar'), processors=[ResizeToFill(500, 500)] ,blank=False, null=True, format='JPEG', + options={'quality': 75}, ) cover = models.ImageField(_('cover'), blank=False, null=True) bio = models.CharField(_('bio'), max_length=settings.PROFILE_BIO_MAX_LENGTH, blank=False, null=True) url = models.URLField(_('url'), blank=False, null=True) diff --git a/openbook_common/migrations/0009_auto_20190209_1631.py b/openbook_common/migrations/0009_auto_20190209_1631.py new file mode 100644 index 00000000..96e5a197 --- /dev/null +++ b/openbook_common/migrations/0009_auto_20190209_1631.py @@ -0,0 +1,19 @@ +# Generated by Django 2.1.5 on 2019-02-09 15:31 + +from django.db import migrations +import imagekit.models.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('openbook_common', '0008_auto_20181221_1317'), + ] + + operations = [ + migrations.AlterField( + model_name='emoji', + name='image', + field=imagekit.models.fields.ProcessedImageField(unique=True, upload_to='', verbose_name='image'), + ), + ] diff --git a/openbook_common/models.py b/openbook_common/models.py index 2c8f6347..44682eff 100644 --- a/openbook_common/models.py +++ b/openbook_common/models.py @@ -5,6 +5,7 @@ from django.utils.translation import ugettext_lazy as _ # Create your views here. +from imagekit.models import ProcessedImageField from openbook.settings import COLOR_ATTR_MAX_LENGTH from openbook_common.validators import hex_color_validator @@ -36,7 +37,8 @@ class Emoji(models.Model): # Hex colour. #FFFFFF color = models.CharField(_('color'), max_length=COLOR_ATTR_MAX_LENGTH, blank=False, null=False, validators=[hex_color_validator], unique=False) - image = models.ImageField(_('image'), blank=False, null=False, unique=True) + image = ProcessedImageField(verbose_name=_('image'), blank=False, null=False, unique=True, + format='JPEG', options={'quality': 100}) created = models.DateTimeField(editable=False) order = models.IntegerField(unique=False, default=100) diff --git a/openbook_importer/facebook_archive_parser/zipparser.py b/openbook_importer/facebook_archive_parser/zipparser.py index 8d6143bf..3904a88f 100755 --- a/openbook_importer/facebook_archive_parser/zipparser.py +++ b/openbook_importer/facebook_archive_parser/zipparser.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 +from uuid import uuid4 from json import loads from yaml import safe_load from hashlib import sha3_256 from zipfile import PyZipFile -from os import access, R_OK, path -from tempfile import TemporaryDirectory +from os import remove, rmdir +from os import access, R_OK, path, mkdir from magic import from_buffer @@ -14,15 +15,23 @@ class profile_import(object): friends = False albums = False - messages = False posts = False - def __init__(self, friends, albums, messages, posts): + def __init__(self, friends, albums, posts, files): self.friends = friends self.albums = albums - self.messages = messages self.posts = posts + self.files = files + self.delete = self.delete_files + + def delete_files(self): + + for file in self.files: + remove(file) + + root = file.split('/') + rmdir(path.join(root[0], root[1])) class zip_parser(): @@ -32,9 +41,10 @@ class zip_parser(): def __init__(self, filename): zipf = PyZipFile(filename) - # test this with corrupt zipfile zipf.testzip() size = self._get_extracted_zipsize(zipf) + self.files = set() + self.temp = self._create_temporary_directory('media') # if size > 1gb if size > 1000000000: @@ -42,10 +52,16 @@ def __init__(self, filename): friends = self._extract_friends(zipf) albums = self._extract_albums(zipf) - messages = self._extract_messages(zipf) posts = self._extract_posts(zipf) - self.profile = profile_import(friends, albums, messages, posts) + self.profile = profile_import(friends, albums, posts, self.files) + + def _create_temporary_directory(self, directory): + + directory_path = path.join(directory, str(uuid4())) + mkdir(directory_path) + + return directory_path def _file_access(self, filename): @@ -140,12 +156,13 @@ def _write_file_to_dir(self, dir_name, item, zipf): with open(i_path, 'wb+') as fd: fd.write(zipf.read(item)) - def _get_fd_from_file(self, dir_name, itype, item, zipf, mode='r'): + def _get_fd_from_file(self, dir_name, itype, item, zipf): self._write_file_to_dir(dir_name, item, zipf) name = item.split('/')[-1] - fd = open(path.join(dir_name, name), mode) + fd = path.join(dir_name, name) + self.files.add(fd) return((name, fd)) @@ -179,15 +196,12 @@ def _extract_albums(self, zipf): for album in album_defs: albums.append(self._parse_album_json(album, zipf)) - temp = TemporaryDirectory(dir='media') - for album in albums: for value in album.values(): for file in (value['photos']): - file['uri'] = self._get_fd_from_file(temp.name, + file['uri'] = self._get_fd_from_file(self.temp, 'photos_and_videos', - file['uri'], zipf, - mode='rb') + file['uri'], zipf) return albums @@ -217,20 +231,24 @@ def _extract_friends(self, zipf): return(friends_hash) + def _is_hypertext_link(self, uri): + + return uri.startswith('http') + def _parse_message(self, zipf, message): json = loads(self._read_file_from_zip(zipf, message)) - temp = TemporaryDirectory(dir='media') - if 'messages' in json.keys(): for m in json['messages']: if 'photos' in m.keys(): for p in m['photos']: - p['uri'] = self._get_fd_from_file(temp.name, - 'messages', - p['uri'], zipf) + + if not self._is_hypertext_link(p['uri']): + p['uri'] = self._get_fd_from_file(self.temp, + 'messages', + p['uri'], zipf) else: raise KeyError('key messages not found in json') @@ -248,19 +266,16 @@ def _extract_messages(self, zipf): def _has_attachment(self, zipf, post): - temp = TemporaryDirectory(dir='media') - if 'attachments' in post.keys(): for attachment in post['attachments']: if 'data' in attachment.keys(): for item in attachment['data']: if 'media' in item.keys(): media = item['media'] - uri = self._get_fd_from_file(temp.name, + uri = self._get_fd_from_file(self.temp, media['uri']. split('/')[0], - media['uri'], zipf, - mode='rb') + media['uri'], zipf) media['uri'] = uri if 'media_metadata' in media: diff --git a/openbook_importer/migrations/0001_initial.py b/openbook_importer/migrations/0001_initial.py new file mode 100644 index 00000000..77b88e7e --- /dev/null +++ b/openbook_importer/migrations/0001_initial.py @@ -0,0 +1,31 @@ +# Generated by Django 2.1.5 on 2019-01-11 17:42 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('openbook_posts', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Import', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(editable=False)), + ], + ), + migrations.CreateModel( + name='ImportedPost', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('data_import', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='imported_posts', to='openbook_importer.Import')), + ('post', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='openbook_posts.Post')), + ], + ), + ] diff --git a/openbook_importer/migrations/0002_importedfriend.py b/openbook_importer/migrations/0002_importedfriend.py new file mode 100644 index 00000000..f6e071a1 --- /dev/null +++ b/openbook_importer/migrations/0002_importedfriend.py @@ -0,0 +1,25 @@ +# Generated by Django 2.1.5 on 2019-01-18 19:57 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('openbook_importer', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='ImportedFriend', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('friend_hash', models.CharField(max_length=64, unique=True)), + ('data_import', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='friend_import', to='openbook_importer.Import')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='imported_friends', to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/openbook_importer/migrations/0003_auto_20190125_1216.py b/openbook_importer/migrations/0003_auto_20190125_1216.py new file mode 100644 index 00000000..2c86a310 --- /dev/null +++ b/openbook_importer/migrations/0003_auto_20190125_1216.py @@ -0,0 +1,34 @@ +# Generated by Django 2.1.5 on 2019-01-25 11:16 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('openbook_importer', '0002_importedfriend'), + ] + + operations = [ + migrations.RemoveField( + model_name='importedfriend', + name='data_import', + ), + migrations.RemoveField( + model_name='importedfriend', + name='user', + ), + migrations.AddField( + model_name='importedfriend', + name='user1', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='imported_friends', to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='importedfriend', + name='user2', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='connected_friend', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/openbook_importer/migrations/0004_import_user.py b/openbook_importer/migrations/0004_import_user.py new file mode 100644 index 00000000..6aa877f9 --- /dev/null +++ b/openbook_importer/migrations/0004_import_user.py @@ -0,0 +1,21 @@ +# Generated by Django 2.1.5 on 2019-02-02 12:20 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('openbook_importer', '0003_auto_20190125_1216'), + ] + + operations = [ + migrations.AddField( + model_name='import', + name='user', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='import_owner', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/openbook_importer/migrations/0005_auto_20190202_1644.py b/openbook_importer/migrations/0005_auto_20190202_1644.py new file mode 100644 index 00000000..f1edc686 --- /dev/null +++ b/openbook_importer/migrations/0005_auto_20190202_1644.py @@ -0,0 +1,20 @@ +# Generated by Django 2.1.5 on 2019-02-02 15:44 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('openbook_importer', '0004_import_user'), + ] + + operations = [ + migrations.AlterField( + model_name='import', + name='user', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/openbook_importer/migrations/0006_auto_20190202_1647.py b/openbook_importer/migrations/0006_auto_20190202_1647.py new file mode 100644 index 00000000..46e3e888 --- /dev/null +++ b/openbook_importer/migrations/0006_auto_20190202_1647.py @@ -0,0 +1,20 @@ +# Generated by Django 2.1.5 on 2019-02-02 15:47 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('openbook_importer', '0005_auto_20190202_1644'), + ] + + operations = [ + migrations.AlterField( + model_name='import', + name='user', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/openbook_importer/migrations/0007_auto_20190202_1848.py b/openbook_importer/migrations/0007_auto_20190202_1848.py new file mode 100644 index 00000000..88d0795d --- /dev/null +++ b/openbook_importer/migrations/0007_auto_20190202_1848.py @@ -0,0 +1,20 @@ +# Generated by Django 2.1.5 on 2019-02-02 17:48 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('openbook_importer', '0006_auto_20190202_1647'), + ] + + operations = [ + migrations.AlterField( + model_name='import', + name='user', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/openbook_importer/migrations/0008_import_uuid.py b/openbook_importer/migrations/0008_import_uuid.py new file mode 100644 index 00000000..9ac510a3 --- /dev/null +++ b/openbook_importer/migrations/0008_import_uuid.py @@ -0,0 +1,19 @@ +# Generated by Django 2.1.5 on 2019-02-09 15:31 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('openbook_importer', '0007_auto_20190202_1848'), + ] + + operations = [ + migrations.AddField( + model_name='import', + name='uuid', + field=models.UUIDField(default=uuid.uuid4, editable=False), + ), + ] diff --git a/openbook_importer/migrations/0009_remove_import_uuid.py b/openbook_importer/migrations/0009_remove_import_uuid.py new file mode 100644 index 00000000..3d0ccd3b --- /dev/null +++ b/openbook_importer/migrations/0009_remove_import_uuid.py @@ -0,0 +1,17 @@ +# Generated by Django 2.1.5 on 2019-02-09 15:38 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('openbook_importer', '0008_import_uuid'), + ] + + operations = [ + migrations.RemoveField( + model_name='import', + name='uuid', + ), + ] diff --git a/openbook_importer/migrations/0010_import_uuid.py b/openbook_importer/migrations/0010_import_uuid.py new file mode 100644 index 00000000..f9b9d1b1 --- /dev/null +++ b/openbook_importer/migrations/0010_import_uuid.py @@ -0,0 +1,19 @@ +# Generated by Django 2.1.5 on 2019-02-09 15:38 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('openbook_importer', '0009_remove_import_uuid'), + ] + + operations = [ + migrations.AddField( + model_name='import', + name='uuid', + field=models.UUIDField(default=uuid.uuid4, editable=False), + ), + ] diff --git a/openbook_importer/migrations/__init__.py b/openbook_importer/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openbook_importer/models.py b/openbook_importer/models.py new file mode 100644 index 00000000..5cac309d --- /dev/null +++ b/openbook_importer/models.py @@ -0,0 +1,101 @@ +from uuid import uuid4 + +from django.db import models +from django.utils import timezone + +from openbook_auth.models import User +from openbook_posts.models import Post + + +class Import(models.Model): + + created = models.DateTimeField(editable=False) + user = models.ForeignKey(User, on_delete=models.CASCADE, + related_name='imports', null=True) + uuid = models.UUIDField(editable=False, default=uuid4) + + @classmethod + def create_import(cls, user): + imported = Import.objects.create(user=user) + + return imported + + @property + def posts(self): + return ImportedPost.objects.filter(data_import_id=self.pk).count() + + def save(self, *args, **kwargs): + ''' On save, update timestamps ''' + if not self.id and not self.created: + self.created = timezone.now() + + return super(Import, self).save(*args, **kwargs) + + +class ImportedPost(models.Model): + + data_import = models.ForeignKey(Import, on_delete=models.CASCADE, + related_name='imported_posts') + post = models.ForeignKey(Post, on_delete=models.CASCADE) + + @classmethod + def create_imported_post(cls, post, data_import): + imported_post = ImportedPost.objects.create(post=post, + data_import=data_import) + + return imported_post + + +class ImportedFriend(models.Model): + + # check if both uid fields are null, delete row + friend_hash = models.CharField(max_length=64, unique=True) + user1 = models.ForeignKey(User, on_delete=models.SET_NULL, + related_name='imported_friends', + null=True) + user2 = models.ForeignKey(User, on_delete=models.SET_NULL, + related_name='connected_friend', + null=True) + + @classmethod + def find_friend(cls, friend_hash, user): + + friend = ImportedFriend.objects.filter(friend_hash=friend_hash) + + if friend.exists(): + friend = friend[0] + + if friend.user1_id == user.pk and friend.user2_id: + return True + + elif friend.user2_id == user.pk and friend.user1_id: + return True + + elif friend.user1_id == user.pk and not friend.user2_id: + return True + + elif friend.user2_id == user.pk and not friend.user1_id: + return True + + elif not friend.user1_id and friend.user2_id != user.pk: + friend.user1 = user + friend.save() + + return True + + elif not friend.user2_id and friend.user1_id != user.pk: + friend.user2 = user + friend.save() + + return True + + else: + return False + + @classmethod + def create_imported_friend(cls, friend_hash, user1): + imported_friend = ImportedFriend.objects.create( + friend_hash=friend_hash, + user1=user1) + + return imported_friend diff --git a/openbook_importer/serializers.py b/openbook_importer/serializers.py index 66bba1fb..2eefbf09 100644 --- a/openbook_importer/serializers.py +++ b/openbook_importer/serializers.py @@ -1,7 +1,19 @@ from rest_framework import serializers +from openbook_importer.models import Import class ZipfileSerializer(serializers.Serializer): - serializers.FileField(max_length=20, required=True, - allow_empty_file=False) + file = serializers.FileField(max_length=1000000000, required=True, + allow_empty_file=False) + + +class ImportSerializer(serializers.ModelSerializer): + + class Meta: + model = Import + fields = ( + 'uuid', + 'created', + 'posts' + ) diff --git a/openbook_importer/tests/facebook-jayjay6.zip b/openbook_importer/tests/facebook-jayjay6.zip new file mode 100644 index 00000000..14d60979 Binary files /dev/null and b/openbook_importer/tests/facebook-jayjay6.zip differ diff --git a/openbook_importer/tests/tests.py b/openbook_importer/tests/tests.py index 3a6f4a00..12740711 100644 --- a/openbook_importer/tests/tests.py +++ b/openbook_importer/tests/tests.py @@ -3,6 +3,7 @@ from rest_framework.test import APITestCase from openbook_common.tests.helpers import make_user +from openbook_importer.models import ImportedPost, Import, ImportedFriend from openbook_common.tests.helpers import make_authentication_headers_for_user @@ -78,3 +79,207 @@ def test_upload_file_duplicate(self): response = self.client.get(reverse('posts'), **headers) self.assertEqual(len(response.json()), number_of_posts) + + def test_postimport_entries(self): + """ + Uploading archive test ImportedPost table should contain 9 entries + """ + + user = make_user() + headers = make_authentication_headers_for_user(user) + + with open('openbook_importer/tests/facebook-jaybeenote5.zip', + 'rb') as fd: + for i in range(0, 2): + response = self.client.post(reverse('uploads'), {'file': fd}, + **headers) + fd.seek(0) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + number_of_entries = 9 + + self.assertEqual(len(ImportedPost.objects.all()), number_of_entries) + + def test_import_entries(self): + """ + Uploading archive Import table should contain 1 zip entry + """ + + user = make_user() + headers = make_authentication_headers_for_user(user) + + with open('openbook_importer/tests/facebook-jaybeenote5.zip', + 'rb') as fd: + for i in range(0, 2): + response = self.client.post(reverse('uploads'), {'file': fd}, + **headers) + fd.seek(0) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + number_of_entries = 1 + + self.assertEqual(len(Import.objects.all()), number_of_entries) + + def test_friendimport_entries(self): + """ + Uploading archive Friend table should contain 2 zip entries + """ + + user = make_user() + headers = make_authentication_headers_for_user(user) + + with open('openbook_importer/tests/facebook-jaybeenote5.zip', + 'rb') as fd: + for i in range(0, 2): + response = self.client.post(reverse('uploads'), {'file': fd}, + **headers) + fd.seek(0) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + number_of_entries = 2 + + self.assertEqual(len(ImportedFriend.objects.all()), number_of_entries) + + def test_findfriend_entries(self): + """ + Uploading the archive as a new user, would lead to a connection + between user1 and user2 + """ + + user = make_user() + headers = make_authentication_headers_for_user(user) + + with open('openbook_importer/tests/facebook-jaybeenote5.zip', + 'rb') as fd: + response = self.client.post(reverse('uploads'), {'file': fd}, + **headers) + + user2 = make_user() + headers = make_authentication_headers_for_user(user2) + + with open('openbook_importer/tests/facebook-jayjay6.zip', + 'rb') as fd: + response = self.client.post(reverse('uploads'), {'file': fd}, + **headers) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + number_of_entries = 2 + + friend_objects = ImportedFriend.objects.all() + self.assertEqual(len(friend_objects), number_of_entries) + + for friend_object in friend_objects: + self.assertTrue(friend_object.user1_id == 1) + self.assertTrue(friend_object.user2_id == 2) + + +class ImportedItemTest(APITestCase): + + def test_delete_importedposts(self): + """ + Deleting a previously imported import, should remove all corresponding + posts. + """ + + user = make_user() + headers = make_authentication_headers_for_user(user) + + with open('openbook_importer/tests/facebook-jayjay6.zip', + 'rb') as fd: + self.client.post(reverse('uploads'), {'file': fd}, + **headers) + + uuid = user.imports.filter(id=1)[0].uuid + self.client.delete(reverse('imported-archive', + kwargs={'archive_id': str(uuid)}), + **headers) + + self.assertEqual(user.imports.all().count(), 0) + self.assertEqual(len(ImportedPost.objects.all()), 0) + + def test_get_imports(self): + + """ + Retrieving imports should result in a `uuid`, `created` and `posts` + field in a serialized JSON response + """ + + user = make_user() + headers = make_authentication_headers_for_user(user) + + with open('openbook_importer/tests/facebook-jayjay6.zip', + 'rb') as fd: + self.client.post(reverse('uploads'), {'file': fd}, + **headers) + + keys = ['uuid', 'created', 'posts'] + + response = self.client.get(reverse('imported-archives'), + **headers).json() + + for archive in response: + for key in archive.keys(): + + self.assertTrue(key in keys) + self.assertIsNotNone(archive[key]) + + def test_get_import(self): + + """ + Retrieving an import should result in a `uuid`, `created` and `posts` + field in a serialized JSON response + """ + + user = make_user() + headers = make_authentication_headers_for_user(user) + + with open('openbook_importer/tests/facebook-jayjay6.zip', + 'rb') as fd: + self.client.post(reverse('uploads'), {'file': fd}, + **headers) + + keys = ['uuid', 'created', 'posts'] + + uuid = user.imports.filter(id=1)[0].uuid + + response = self.client.get(reverse('imported-archive', + kwargs={'archive_id': str(uuid)}), + **headers).json()[0] + + for key in response.keys(): + + self.assertTrue(key in keys) + self.assertIsNotNone(response[key]) + + def test_get_unauthorized_imports(self): + + """ + Retrieving import with other user UUID should return 404 + """ + + user = make_user() + headers = make_authentication_headers_for_user(user) + + with open('openbook_importer/tests/facebook-jaybeenote5.zip', + 'rb') as fd: + response = self.client.post(reverse('uploads'), {'file': fd}, + **headers) + + user2 = make_user() + headers = make_authentication_headers_for_user(user2) + + with open('openbook_importer/tests/facebook-jayjay6.zip', + 'rb') as fd: + response = self.client.post(reverse('uploads'), {'file': fd}, + **headers) + + uuid = user.imports.filter(id=1)[0].uuid + response = self.client.get(reverse('imported-archive', + kwargs={'archive_id': str(uuid)}), + **headers) + + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) diff --git a/openbook_importer/views.py b/openbook_importer/views.py index df5bb652..82cc2d98 100644 --- a/openbook_importer/views.py +++ b/openbook_importer/views.py @@ -1,8 +1,9 @@ +import logging from datetime import datetime from json import JSONDecodeError +from django.db import transaction from rest_framework import status -from openbook_auth.models import User from openbook_posts.models import Post from rest_framework.views import APIView from rest_framework.response import Response @@ -10,9 +11,14 @@ from django.utils.dateparse import parse_datetime from rest_framework.permissions import IsAuthenticated from django.utils.translation import ugettext_lazy as _ +from rest_framework.exceptions import ValidationError, NotFound -from openbook_importer.serializers import ZipfileSerializer +from openbook_common.responses import ApiMessageResponse +from openbook_importer.models import Import, ImportedPost, ImportedFriend from openbook_importer.facebook_archive_parser.zipparser import zip_parser +from openbook_importer.serializers import ZipfileSerializer, ImportSerializer + +log = logging.getLogger('security') class ImportItem(APIView): @@ -20,62 +26,137 @@ class ImportItem(APIView): permission_classes = (IsAuthenticated,) def post(self, request): + serializer = ZipfileSerializer(data=request.FILES) serializer.is_valid(raise_exception=True) + zipfile = serializer.validated_data.get('file') - zipfile = request.FILES['file'] + new_friends = False + new_posts = False try: p = zip_parser(zipfile) except FileNotFoundError: - return self._return_invalid() + raise ValidationError( + _('Invalid archive!'), + ) except JSONDecodeError: - return self._return_invalid() + raise ValidationError( + _('Invalid archive!'), + ) except TypeError: - return self._return_malicious() + log.info("Potentially malicious import prevented: " + f"{request.user.pk}") + raise ValidationError( + _('Invalid archive!'), + ) if p.profile.posts: - self.save_posts(p.profile.posts, request.user) + new_posts = self._create_posts(p.profile.posts, request.user) + + if p.profile.friends: + new_friends = self._get_friends(p.profile.friends, request.user) + + self.process_imports(new_posts, new_friends, request.user) + p.profile.delete() + + return Response(status=status.HTTP_200_OK) + + def process_imports(self, new_posts, new_friends, user): + + if new_posts or new_friends: + data_import = Import.create_import(user) + + if new_posts: + self.save_posts(new_posts, user, data_import) + + if new_friends: + self.save_friends(new_friends, user, data_import) + + def save_posts(self, new_posts, user, data_import): + + for new_post in new_posts: + text, image, created = new_post - return Response({ - 'message': _('done') - }, status=status.HTTP_200_OK) + with transaction.atomic(): + post = user.create_post(text=text, image=image, + created=created) + ImportedPost.create_imported_post(post, data_import) - def save_posts(self, posts, user): + def save_friends(self, new_hashes, user, data_import): - for post in posts: + for friend_hash in new_hashes: + + with transaction.atomic(): + ImportedFriend.create_imported_friend(friend_hash, user) + + def _create_posts(self, posts_data, user): + + new_posts = [] + + # Decrease complexity + + for post_data in posts_data: image = None images = None text = None - timestamp = post['timestamp'] + timestamp = post_data['timestamp'] + created = datetime.fromtimestamp(timestamp) created = parse_datetime(created.strftime('%Y-%m-%d %T+00:00')) - if 'attachments' in post.keys(): - images = self._get_media_content(post) + if 'attachments' in post_data.keys(): + images = self._get_media_content(post_data) - if 'data' in post.keys() and len(post['data']) != 0: - text = post['data'][0]['post'] + if 'data' in post_data.keys() and len(post_data['data']) != 0: + text = post_data['data'][0]['post'] if images: + # Currently we only support having one post per image image = images[0] if 'text' in image.keys(): text = image['text'] - image = ImageFile(image['file']) + image = ImageFile(open(image['file'], 'rb')) + + if not self._post_exists(user.pk, text=text, + created=created): + new_posts.append((text, image, created)) + + return new_posts + + def _post_exists(self, creator, text, created): + return Post.objects.filter(creator=creator, text=text, + created=created).exists() + + def _friend_link_exists(self, friend_hash, user): - if not Post.objects.filter(creator=user.pk, text=text, created=created).exists(): - user.create_post(text=text, image=image, created=created) + friend = ImportedFriend.find_friend(friend_hash, user) + + return friend + + def _get_friends(self, friend_hashes, user): + + new_friends = [] + + for friend_hash in friend_hashes: + if not self._friend_link_exists(friend_hash, user): + new_friends.append(friend_hash) + + return new_friends def _get_media_content(self, post): + # add video + images = [] image = {} + # simplify for attachment in post['attachments']: for data in attachment['data']: image['file'] = data['media']['uri'][1] @@ -88,16 +169,39 @@ def _get_media_content(self, post): return images - def _return_invalid(self): - return Response({ - 'message':_('invalid archive') - }, status=status.HTTP_400_BAD_REQUEST) +class ImportedItem(APIView): + + permission_classes = (IsAuthenticated,) + + def get(self, request, archive_id): + + archive = request.user.imports.filter(uuid=archive_id) + + if archive.exists(): + serialized = ImportSerializer(archive, many=True) + + else: + raise NotFound( + _('Archive does not exist!'), + ) + + return Response(serialized.data, status=status.HTTP_200_OK) + + def delete(self, request, archive_id): + + request.user.delete_archive_with_id(archive_id) + + return ApiMessageResponse(_('Done!')) + + +class ImportedItems(APIView): + + permission_classes = (IsAuthenticated,) + + def get(self, request): - def _return_malicious(self): - # TODO LOG MALICIOUS ATTEMPT - print('---- POTENTIALLY MALICIOUS UPLOAD!!!') + archives = request.user.imports.all() + serialized = ImportSerializer(archives, many=True) - return Response({ - 'message':_('invalid archive') - }, status=status.HTTP_400_BAD_REQUEST) + return Response(serialized.data, status=status.HTTP_200_OK) diff --git a/openbook_posts/migrations/0015_auto_20190209_1631.py b/openbook_posts/migrations/0015_auto_20190209_1631.py new file mode 100644 index 00000000..a536a754 --- /dev/null +++ b/openbook_posts/migrations/0015_auto_20190209_1631.py @@ -0,0 +1,19 @@ +# Generated by Django 2.1.5 on 2019-02-09 15:31 + +from django.db import migrations +import imagekit.models.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('openbook_posts', '0014_auto_20181215_1354'), + ] + + operations = [ + migrations.AlterField( + model_name='postimage', + name='image', + field=imagekit.models.fields.ProcessedImageField(upload_to='./media', verbose_name='image'), + ), + ] diff --git a/openbook_posts/models.py b/openbook_posts/models.py index 0c8c03d0..d10270df 100644 --- a/openbook_posts/models.py +++ b/openbook_posts/models.py @@ -8,6 +8,8 @@ # Create your views here. from rest_framework.exceptions import ValidationError +from imagekit.models import ProcessedImageField +from imagekit.processors import ResizeToFill from django.conf import settings from openbook.storage_backends import S3PrivateMediaStorage @@ -148,7 +150,8 @@ def save(self, *args, **kwargs): class PostImage(models.Model): post = models.OneToOneField(Post, on_delete=models.CASCADE, related_name='image') - image = models.ImageField(_('image'), blank=False, null=False, storage=post_image_storage) + image = ProcessedImageField(verbose_name=_('image'), upload_to=post_image_storage.base_location, format='JPEG', options={'quality': 75}, + blank=False, null=False) class PostVideo(models.Model): diff --git a/requirements.txt b/requirements.txt index ae3f179c..5cde0c33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ certifi==2018.11.29 colorama==0.4.1 coverage==5.0a4 django-amazon-ses==2.0.0 +django-imagekit==4.0.2 django-media-fixtures==0.0.3 django-nose==1.4.6 django-storages==1.7.1