From 5529eac0da852a55658a945eef9596f7fc74bedd Mon Sep 17 00:00:00 2001 From: henrikek Date: Mon, 8 Feb 2021 21:08:27 +0100 Subject: [PATCH 01/13] Add policy to remote --- ESSArch_Core/ip/models.py | 15 +++++++-- ESSArch_Core/ip/serializers.py | 60 ++++++++++++++++++++++++++++++++-- ESSArch_Core/ip/views.py | 3 ++ 3 files changed, 72 insertions(+), 6 deletions(-) diff --git a/ESSArch_Core/ip/models.py b/ESSArch_Core/ip/models.py index 5ddd67fe1..862d8e85e 100644 --- a/ESSArch_Core/ip/models.py +++ b/ESSArch_Core/ip/models.py @@ -1716,12 +1716,21 @@ def get_temp_container_aic_xml_path(self): @retry(retry=retry_if_exception_type(RequestException), reraise=True, stop=stop_after_attempt(5), wait=wait_fixed(60), before_sleep=before_sleep_log(logger, logging.DEBUG)) def update_remote_ip(self, host, session): - from ESSArch_Core.ip.serializers import InformationPackageFromMasterSerializer + from ESSArch_Core.ip.serializers import ( + InformationPackageFromMasterSerializer, + ) remote_ip = urljoin(host, reverse('informationpackage-add-from-master')) data = InformationPackageFromMasterSerializer(instance=self).data - response = session.post(remote_ip, json=data, timeout=10) - response.raise_for_status() + response = None + try: + response = session.post(remote_ip, json=data, timeout=10) + response.raise_for_status() + except RequestException as e: + msg = 'Response: {response}, post_url: {post_url}, post_data: {post_data}'.format( + response=e.response.text, post_url=remote_ip, post_data=data) + logger.error(msg) + raise e @retry(retry=retry_if_exception_type(StorageMediumFull), reraise=True, stop=stop_after_attempt(2), wait=wait_fixed(60), before_sleep=before_sleep_log(logger, logging.DEBUG)) diff --git a/ESSArch_Core/ip/serializers.py b/ESSArch_Core/ip/serializers.py index cac363c42..41cfa757e 100644 --- a/ESSArch_Core/ip/serializers.py +++ b/ESSArch_Core/ip/serializers.py @@ -11,7 +11,8 @@ from ESSArch_Core.api.serializers import DynamicModelSerializer from ESSArch_Core.auth.fields import CurrentUsernameDefault from ESSArch_Core.auth.serializers import GroupSerializer, UserSerializer -from ESSArch_Core.configuration.models import EventType +from ESSArch_Core.configuration.models import EventType, Path, StoragePolicy +from ESSArch_Core.configuration.serializers import StoragePolicySerializer from ESSArch_Core.ip.models import ( Agent, AgentNote, @@ -506,6 +507,36 @@ class Meta: class InformationPackageFromMasterSerializer(serializers.ModelSerializer): aic = InformationPackageAICSerializer(omit=['information_packages']) + policy = StoragePolicySerializer() + organization = serializers.SerializerMethodField() + submission_agreement_data = serializers.SerializerMethodField() + submission_agreement_data_versions = serializers.ListField( + child=serializers.PrimaryKeyRelatedField(read_only=True) + ) + profiles = ProfileIPSerializer(source='profileip_set', many=True) + + def get_organization(self, obj): + try: + return GroupSerializer(obj.org[0].group).data + except AttributeError: + return GroupSerializer(obj.generic_groups.first().group).data + except IndexError: + return None + + def get_submission_agreement_data(self, obj): + if obj.submission_agreement_data is not None: + serializer = SubmissionAgreementIPDataSerializer(obj.submission_agreement_data) + data = serializer.data + else: + data = {'data': {}} + + extra_data = fill_specification_data(ip=obj, sa=obj.submission_agreement) + + for field in getattr(obj.submission_agreement, 'template', []): + if field['key'] in extra_data: + data['data'][field['key']] = extra_data[field['key']] + + return data def create_storage_method(self, data): storage_method_target_set_data = data.pop('storage_method_target_relations') @@ -535,6 +566,26 @@ def create(self, validated_data): aic_data['last_changed_local'] = timezone.now aic, _ = InformationPackage.objects.update_or_create(id=aic_data['id'], defaults=aic_data) + policy_data = validated_data.pop('policy') + storage_method_set_data = policy_data.pop('storage_methods') + + cache_storage_data = policy_data.pop('cache_storage') + ingest_path_data = policy_data.pop('ingest_path') + + cache_storage = self.create_storage_method(cache_storage_data) + ingest_path, _ = Path.objects.update_or_create(entity=ingest_path_data['entity'], defaults=ingest_path_data) + + policy_data['cache_storage'] = cache_storage + policy_data['ingest_path'] = ingest_path + + policy, _ = StoragePolicy.objects.update_or_create(policy_id=policy_data['policy_id'], + defaults=policy_data) + + for storage_method_data in storage_method_set_data: + storage_method = self.create_storage_method(storage_method_data) + policy.storage_methods.add(storage_method) + # add to policy, dummy + request = self.context.get("request") if request and hasattr(request, "user"): user = request.user @@ -542,6 +593,7 @@ def create(self, validated_data): user = User.objects.get(username="system") validated_data['aic'] = aic + validated_data['policy'] = policy validated_data['responsible'] = user validated_data['last_changed_local'] = timezone.now ip, _ = InformationPackage.objects.update_or_create(id=validated_data['id'], defaults=validated_data) @@ -553,12 +605,13 @@ class Meta: fields = ( 'id', 'label', 'object_identifier_value', 'object_size', 'object_path', 'package_type', 'responsible', 'create_date', - 'object_num_items', 'entry_date', 'state', 'status', 'step_state', + 'object_num_items', 'entry_date', 'state', 'archived', 'cached', 'aic', 'generation', 'message_digest', 'message_digest_algorithm', 'content_mets_create_date', 'content_mets_size', 'content_mets_digest_algorithm', 'content_mets_digest', 'package_mets_create_date', 'package_mets_size', 'package_mets_digest_algorithm', 'package_mets_digest', - 'start_date', 'end_date', 'appraisal_date', + 'start_date', 'end_date', 'appraisal_date', 'profiles', 'policy', 'organization', 'submission_agreement', + 'submission_agreement_locked', 'submission_agreement_data', 'submission_agreement_data_versions', ) extra_kwargs = { 'id': { @@ -570,6 +623,7 @@ class Meta: 'validators': [], }, } + validators = [] # Remove a default "unique together" constraint. class NestedInformationPackageSerializer(InformationPackageSerializer): diff --git a/ESSArch_Core/ip/views.py b/ESSArch_Core/ip/views.py index c3fdec5a7..78a430f64 100644 --- a/ESSArch_Core/ip/views.py +++ b/ESSArch_Core/ip/views.py @@ -1235,6 +1235,9 @@ def get_xsds(self, request, pk=None): @transaction.atomic @action(detail=False, methods=['post'], url_path='add-from-master') def add_from_master(self, request, pk=None): + self.logger.debug( + 'ip - views.py - add_from_master - request.data: %s' % (repr(request.data)) + ) serializer = InformationPackageFromMasterSerializer( data=request.data, context={'request': request}, ) From bfb1dc7e4179ab2db0a6e24b985a7aa53e3d8d18 Mon Sep 17 00:00:00 2001 From: henrikek Date: Mon, 8 Feb 2021 22:30:05 +0100 Subject: [PATCH 02/13] Add serializer for SA UUID --- ESSArch_Core/ip/serializers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ESSArch_Core/ip/serializers.py b/ESSArch_Core/ip/serializers.py index 41cfa757e..c3723281e 100644 --- a/ESSArch_Core/ip/serializers.py +++ b/ESSArch_Core/ip/serializers.py @@ -509,6 +509,10 @@ class InformationPackageFromMasterSerializer(serializers.ModelSerializer): aic = InformationPackageAICSerializer(omit=['information_packages']) policy = StoragePolicySerializer() organization = serializers.SerializerMethodField() + submission_agreement = serializers.PrimaryKeyRelatedField( + queryset=SubmissionAgreement.objects.all(), + pk_field=serializers.UUIDField(format='hex_verbose'), + ) submission_agreement_data = serializers.SerializerMethodField() submission_agreement_data_versions = serializers.ListField( child=serializers.PrimaryKeyRelatedField(read_only=True) From b3e87c053126f7b5c841577aed9597af38c2052f Mon Sep 17 00:00:00 2001 From: henrikek Date: Wed, 10 Feb 2021 17:54:05 +0100 Subject: [PATCH 03/13] Fix only 1 ST can be enabled for a SM at a time --- ESSArch_Core/storage/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ESSArch_Core/storage/models.py b/ESSArch_Core/storage/models.py index 4432aeed3..be61dbf4f 100644 --- a/ESSArch_Core/storage/models.py +++ b/ESSArch_Core/storage/models.py @@ -286,7 +286,7 @@ def save(self, *args, **kwargs): if StorageMethodTargetRelation.objects.filter( storage_method=self.storage_method, status=STORAGE_TARGET_STATUS_ENABLED, - ).exists(): + ).count() > 1: raise ValidationError(_('Only 1 target can be enabled for a storage method at a time'),) return super().save(*args, **kwargs) From c09b01a78fafe0d2ab62c43b7ac1b69b66edf774 Mon Sep 17 00:00:00 2001 From: henrikek Date: Wed, 10 Feb 2021 17:57:14 +0100 Subject: [PATCH 04/13] Allow none cache storage in storage policy --- ESSArch_Core/configuration/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ESSArch_Core/configuration/serializers.py b/ESSArch_Core/configuration/serializers.py index 833f5abb9..84327d2b8 100644 --- a/ESSArch_Core/configuration/serializers.py +++ b/ESSArch_Core/configuration/serializers.py @@ -133,7 +133,7 @@ class Meta: class StoragePolicySerializer(serializers.ModelSerializer): - cache_storage = StorageMethodSerializer() + cache_storage = StorageMethodSerializer(allow_null=True) storage_methods = StorageMethodSerializer(many=True) ingest_path = PathSerializer() From 6a792d47262ae4ffa37ea684bbbbd8b3ff80ce07 Mon Sep 17 00:00:00 2001 From: henrikek Date: Wed, 10 Feb 2021 18:00:44 +0100 Subject: [PATCH 05/13] Convert "remote" string exception to Exception --- ESSArch_Core/celery/backends/database.py | 5 ++++- ESSArch_Core/ip/serializers.py | 22 +++++++++++++--------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/ESSArch_Core/celery/backends/database.py b/ESSArch_Core/celery/backends/database.py index 3ee4afdc2..f4ac517af 100644 --- a/ESSArch_Core/celery/backends/database.py +++ b/ESSArch_Core/celery/backends/database.py @@ -101,8 +101,11 @@ def _get_task_meta_for(self, task_id): @classmethod def exception_to_python(cls, exc): - """Convert serialized exception to Python exception.""" + """Convert serialized exception or string to Python exception.""" if exc: + if isinstance(exc, str): + exc = Exception(exc) + if not isinstance(exc, BaseException): exc_module = exc.get('exc_module') if exc_module is None: diff --git a/ESSArch_Core/ip/serializers.py b/ESSArch_Core/ip/serializers.py index c3723281e..663efd9e3 100644 --- a/ESSArch_Core/ip/serializers.py +++ b/ESSArch_Core/ip/serializers.py @@ -508,16 +508,16 @@ class Meta: class InformationPackageFromMasterSerializer(serializers.ModelSerializer): aic = InformationPackageAICSerializer(omit=['information_packages']) policy = StoragePolicySerializer() - organization = serializers.SerializerMethodField() + #organization = serializers.SerializerMethodField() submission_agreement = serializers.PrimaryKeyRelatedField( queryset=SubmissionAgreement.objects.all(), pk_field=serializers.UUIDField(format='hex_verbose'), ) - submission_agreement_data = serializers.SerializerMethodField() - submission_agreement_data_versions = serializers.ListField( - child=serializers.PrimaryKeyRelatedField(read_only=True) - ) - profiles = ProfileIPSerializer(source='profileip_set', many=True) + #submission_agreement_data = serializers.SerializerMethodField() + # submission_agreement_data_versions = serializers.ListField( + # child=serializers.PrimaryKeyRelatedField(read_only=True) + # ) + #profiles = ProfileIPSerializer(source='profileip_set', many=True) def get_organization(self, obj): try: @@ -543,6 +543,9 @@ def get_submission_agreement_data(self, obj): return data def create_storage_method(self, data): + if data is None: + return None + storage_method_target_set_data = data.pop('storage_method_target_relations') storage_method, _ = StorageMethod.objects.update_or_create( id=data['id'], @@ -597,7 +600,7 @@ def create(self, validated_data): user = User.objects.get(username="system") validated_data['aic'] = aic - validated_data['policy'] = policy + #validated_data['policy'] = policy validated_data['responsible'] = user validated_data['last_changed_local'] = timezone.now ip, _ = InformationPackage.objects.update_or_create(id=validated_data['id'], defaults=validated_data) @@ -614,8 +617,9 @@ class Meta: 'message_digest', 'message_digest_algorithm', 'content_mets_create_date', 'content_mets_size', 'content_mets_digest_algorithm', 'content_mets_digest', 'package_mets_create_date', 'package_mets_size', 'package_mets_digest_algorithm', 'package_mets_digest', - 'start_date', 'end_date', 'appraisal_date', 'profiles', 'policy', 'organization', 'submission_agreement', - 'submission_agreement_locked', 'submission_agreement_data', 'submission_agreement_data_versions', + 'start_date', 'end_date', 'appraisal_date', 'policy', 'submission_agreement', + #'start_date', 'end_date', 'appraisal_date', 'profiles', 'policy', 'organization', 'submission_agreement', + #'submission_agreement_locked', 'submission_agreement_data', 'submission_agreement_data_versions', ) extra_kwargs = { 'id': { From b2bb510a645e106f2959815d74e18f3ccbe7844a Mon Sep 17 00:00:00 2001 From: henrikek Date: Mon, 15 Feb 2021 14:11:53 +0100 Subject: [PATCH 06/13] Add storage method in storage policy serializer --- ESSArch_Core/configuration/serializers.py | 47 +++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/ESSArch_Core/configuration/serializers.py b/ESSArch_Core/configuration/serializers.py index 84327d2b8..a3fa17d05 100644 --- a/ESSArch_Core/configuration/serializers.py +++ b/ESSArch_Core/configuration/serializers.py @@ -137,6 +137,53 @@ class StoragePolicySerializer(serializers.ModelSerializer): storage_methods = StorageMethodSerializer(many=True) ingest_path = PathSerializer() + def create_storage_method(self, data): + if data is None: + return None + + storage_method_target_set_data = data.pop('storage_method_target_relations') + storage_method, _ = StorageMethod.objects.update_or_create( + id=data['id'], + defaults=data + ) + + for storage_method_target_data in storage_method_target_set_data: + storage_target_data = storage_method_target_data.pop('storage_target') + storage_target_data.pop('remote_server', None) + storage_target, _ = StorageTarget.objects.update_or_create( + id=storage_target_data['id'], + defaults=storage_target_data + ) + storage_method_target_data['storage_method'] = storage_method + storage_method_target_data['storage_target'] = storage_target + storage_method_target, _ = StorageMethodTargetRelation.objects.update_or_create( + id=storage_method_target_data['id'], + defaults=storage_method_target_data + ) + + return storage_method + + def create(self, validated_data): + storage_method_set_data = validated_data.pop('storage_methods') + cache_storage_data = validated_data.pop('cache_storage') + ingest_path_data = validated_data.pop('ingest_path') + + cache_storage = self.create_storage_method(cache_storage_data) + ingest_path, _ = Path.objects.update_or_create(entity=ingest_path_data['entity'], defaults=ingest_path_data) + + validated_data['cache_storage'] = cache_storage + validated_data['ingest_path'] = ingest_path + + policy, _ = StoragePolicy.objects.update_or_create(policy_id=validated_data['policy_id'], + defaults=validated_data) + + for storage_method_data in storage_method_set_data: + storage_method = self.create_storage_method(storage_method_data) + policy.storage_methods.add(storage_method) + # add to policy, dummy + + return policy + class Meta: model = StoragePolicy fields = ( From 441aa69ad21252dccd62a6bac04707e9d3a265c1 Mon Sep 17 00:00:00 2001 From: henrikek Date: Mon, 15 Feb 2021 18:03:14 +0100 Subject: [PATCH 07/13] Fix linting - flake 8 --- ESSArch_Core/ip/serializers.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ESSArch_Core/ip/serializers.py b/ESSArch_Core/ip/serializers.py index 663efd9e3..9666481ee 100644 --- a/ESSArch_Core/ip/serializers.py +++ b/ESSArch_Core/ip/serializers.py @@ -508,16 +508,16 @@ class Meta: class InformationPackageFromMasterSerializer(serializers.ModelSerializer): aic = InformationPackageAICSerializer(omit=['information_packages']) policy = StoragePolicySerializer() - #organization = serializers.SerializerMethodField() + # organization = serializers.SerializerMethodField() submission_agreement = serializers.PrimaryKeyRelatedField( queryset=SubmissionAgreement.objects.all(), pk_field=serializers.UUIDField(format='hex_verbose'), ) - #submission_agreement_data = serializers.SerializerMethodField() + # submission_agreement_data = serializers.SerializerMethodField() # submission_agreement_data_versions = serializers.ListField( # child=serializers.PrimaryKeyRelatedField(read_only=True) # ) - #profiles = ProfileIPSerializer(source='profileip_set', many=True) + # profiles = ProfileIPSerializer(source='profileip_set', many=True) def get_organization(self, obj): try: @@ -600,7 +600,7 @@ def create(self, validated_data): user = User.objects.get(username="system") validated_data['aic'] = aic - #validated_data['policy'] = policy + # validated_data['policy'] = policy validated_data['responsible'] = user validated_data['last_changed_local'] = timezone.now ip, _ = InformationPackage.objects.update_or_create(id=validated_data['id'], defaults=validated_data) @@ -618,8 +618,8 @@ class Meta: 'content_mets_create_date', 'content_mets_size', 'content_mets_digest_algorithm', 'content_mets_digest', 'package_mets_create_date', 'package_mets_size', 'package_mets_digest_algorithm', 'package_mets_digest', 'start_date', 'end_date', 'appraisal_date', 'policy', 'submission_agreement', - #'start_date', 'end_date', 'appraisal_date', 'profiles', 'policy', 'organization', 'submission_agreement', - #'submission_agreement_locked', 'submission_agreement_data', 'submission_agreement_data_versions', + # 'start_date', 'end_date', 'appraisal_date', 'profiles', 'policy', 'organization', 'submission_agreement', + # 'submission_agreement_locked', 'submission_agreement_data', 'submission_agreement_data_versions', ) extra_kwargs = { 'id': { From 4c158676e597c9decf7fe8b3d9fa38f090d8afe5 Mon Sep 17 00:00:00 2001 From: henrikek Date: Wed, 17 Feb 2021 00:53:05 +0100 Subject: [PATCH 08/13] Fix create_from_remote_copy --- ESSArch_Core/configuration/serializers.py | 1 + ESSArch_Core/storage/models.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ESSArch_Core/configuration/serializers.py b/ESSArch_Core/configuration/serializers.py index a3fa17d05..e5df2b8bf 100644 --- a/ESSArch_Core/configuration/serializers.py +++ b/ESSArch_Core/configuration/serializers.py @@ -200,6 +200,7 @@ class Meta: ) extra_kwargs = { 'id': { + 'read_only': False, 'validators': [], }, 'policy_id': { diff --git a/ESSArch_Core/storage/models.py b/ESSArch_Core/storage/models.py index be61dbf4f..581d9aff5 100644 --- a/ESSArch_Core/storage/models.py +++ b/ESSArch_Core/storage/models.py @@ -586,7 +586,10 @@ def create_from_remote_copy(cls, host, session, object_id): data = r.json() data.pop('location_status_display', None) data.pop('status_display', None) - data['storage_target_id'] = data.pop('storage_target') + if data.get('storage_target') is not None: + data['storage_target'] = StorageTarget.objects.get( + pk=data['storage_target'].pop('id'), + ) if data.get('tape_drive') is not None: data['tape_drive'] = TapeDrive.create_from_remote_copy( host, session, data['tape_drive'], create_storage_medium=False From 572231fe6618be911827b0fab4fe3afcc3faacac Mon Sep 17 00:00:00 2001 From: henrikek Date: Wed, 17 Feb 2021 23:50:36 +0100 Subject: [PATCH 09/13] Get storage-policy data when import SA --- .../static/frontend/scripts/controllers/ImportCtrl.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ESSArch_Core/frontend/static/frontend/scripts/controllers/ImportCtrl.js b/ESSArch_Core/frontend/static/frontend/scripts/controllers/ImportCtrl.js index 46b4cc06f..66d8da111 100644 --- a/ESSArch_Core/frontend/static/frontend/scripts/controllers/ImportCtrl.js +++ b/ESSArch_Core/frontend/static/frontend/scripts/controllers/ImportCtrl.js @@ -70,6 +70,13 @@ export default class ImportCtrl { }); }) ); + } else if (key === 'policy') { + promises.push( + $http.get(vm.url + '/api/storage-policies/' + sa[key] + '/', {headers: headers}).then(function (response) { + const data = response.data; + console.log('Policy key: ' + sa[key] + ' data: ' + data); + }) + ); } else { } } From 966740be20eda57bc093761e3a6abd0c9966fca9 Mon Sep 17 00:00:00 2001 From: henrikek Date: Wed, 24 Feb 2021 18:38:11 +0100 Subject: [PATCH 10/13] Include storage policy when import SA --- ESSArch_Core/configuration/serializers.py | 24 ++++++++++++++ .../static/frontend/lang/en/import.ts | 3 ++ .../static/frontend/lang/sv/import.ts | 3 ++ .../scripts/components/ImportComponent.js | 1 + .../scripts/controllers/ImportCtrl.js | 32 +++++++++++++++++-- .../controllers/OverwriteModalInstanceCtrl.js | 14 +++++++- .../modules/essarch.controllers.module.js | 1 + .../scripts/services/storagePolicy.js | 18 ++++++++++- .../modals/storagePolicy-exists-modal.html | 19 +++++++++++ ESSArch_Core/profiles/admin.py | 2 +- ESSArch_Core/profiles/models.py | 1 + 11 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 ESSArch_Core/frontend/static/frontend/views/modals/storagePolicy-exists-modal.html diff --git a/ESSArch_Core/configuration/serializers.py b/ESSArch_Core/configuration/serializers.py index e5df2b8bf..301fc91ca 100644 --- a/ESSArch_Core/configuration/serializers.py +++ b/ESSArch_Core/configuration/serializers.py @@ -34,6 +34,7 @@ Site, StoragePolicy, ) +from ESSArch_Core.exceptions import Conflict from ESSArch_Core.storage.models import ( StorageMethod, StorageMethodTargetRelation, @@ -184,6 +185,29 @@ def create(self, validated_data): return policy + def update(self, instance, validated_data): + storage_method_set_data = validated_data.pop('storage_methods') + cache_storage_data = validated_data.pop('cache_storage') + ingest_path_data = validated_data.pop('ingest_path') + + cache_storage = self.create_storage_method(cache_storage_data) + ingest_path, _ = Path.objects.update_or_create(entity=ingest_path_data['entity'], defaults=ingest_path_data) + + validated_data['cache_storage'] = cache_storage + validated_data['ingest_path'] = ingest_path + + for storage_method_data in storage_method_set_data: + storage_method = self.create_storage_method(storage_method_data) + instance.storage_methods.add(storage_method) + + return super().update(instance, validated_data) + + def validate(self, data): + if self.instance is None and StoragePolicy.objects.filter(pk=data.get('id')).exists(): + raise Conflict('Storage policy already exists') + + return data + class Meta: model = StoragePolicy fields = ( diff --git a/ESSArch_Core/frontend/static/frontend/lang/en/import.ts b/ESSArch_Core/frontend/static/frontend/lang/en/import.ts index 5e1a13534..768bd7371 100644 --- a/ESSArch_Core/frontend/static/frontend/lang/en/import.ts +++ b/ESSArch_Core/frontend/static/frontend/lang/en/import.ts @@ -18,6 +18,9 @@ export default ($translateProvider: ng.translate.ITranslateProvider) => { SA_EXISTS_DESC: 'Submission agreement with same ID already exists. Would you like to overwrite it?', SA_IMPORTED: 'Submission agreement "{{name}}" has been imported. \nID: {{id}}', SA_IS_PUBLISHED_CANNOT_BE_OVERWRITTEN: 'Submission Agreement {{name}} is Published and can not be overwritten', + STORAGEPOLICY_EXISTS: 'Storage policy exists', + STORAGEPOLICY_EXISTS_DESC: 'Storage policy with same ID already exists. Would you like to overwrite it?', + STORAGEPOLICY_IMPORTED: 'Storage policy: "{{name}}" has been imported. \nID: {{id}}', }, }); }; diff --git a/ESSArch_Core/frontend/static/frontend/lang/sv/import.ts b/ESSArch_Core/frontend/static/frontend/lang/sv/import.ts index 6318b5786..4153a7d64 100644 --- a/ESSArch_Core/frontend/static/frontend/lang/sv/import.ts +++ b/ESSArch_Core/frontend/static/frontend/lang/sv/import.ts @@ -19,6 +19,9 @@ export default ($translateProvider: ng.translate.ITranslateProvider) => { SA_IMPORTED: 'Leveransöverenskommelse "{{name}}" har importerats. \nID: {{id}}', SA_IS_PUBLISHED_CANNOT_BE_OVERWRITTEN: 'Leveransöverenskommelse {{name}} är publicerad och kan inte skrivas över', + STORAGEPOLICY_EXISTS: 'Lagringsregelverk finns redan', + STORAGEPOLICY_EXISTS_DESC: 'Lagringsregelverk med samma ID finns redan. VIll du skriva över den?', + STORAGEPOLICY_IMPORTED: 'Lagringsregelverk: "{{name}}" har importerats. \nID: {{id}}', }, }); }; diff --git a/ESSArch_Core/frontend/static/frontend/scripts/components/ImportComponent.js b/ESSArch_Core/frontend/static/frontend/scripts/components/ImportComponent.js index bb2373318..bd0dcbfe8 100644 --- a/ESSArch_Core/frontend/static/frontend/scripts/components/ImportComponent.js +++ b/ESSArch_Core/frontend/static/frontend/scripts/components/ImportComponent.js @@ -13,6 +13,7 @@ export default { 'Notifications', '$uibModal', '$translate', + 'StoragePolicy', controller, ], controllerAs: 'vm', diff --git a/ESSArch_Core/frontend/static/frontend/scripts/controllers/ImportCtrl.js b/ESSArch_Core/frontend/static/frontend/scripts/controllers/ImportCtrl.js index 66d8da111..c6921aa4c 100644 --- a/ESSArch_Core/frontend/static/frontend/scripts/controllers/ImportCtrl.js +++ b/ESSArch_Core/frontend/static/frontend/scripts/controllers/ImportCtrl.js @@ -1,5 +1,5 @@ export default class ImportCtrl { - constructor($q, $rootScope, $scope, $http, IP, Profile, SA, Notifications, $uibModal, $translate) { + constructor($q, $rootScope, $scope, $http, IP, Profile, SA, Notifications, $uibModal, $translate, StoragePolicy) { const vm = this; $scope.angular = angular; vm.loadingSas = false; @@ -74,7 +74,17 @@ export default class ImportCtrl { promises.push( $http.get(vm.url + '/api/storage-policies/' + sa[key] + '/', {headers: headers}).then(function (response) { const data = response.data; - console.log('Policy key: ' + sa[key] + ' data: ' + data); + return StoragePolicy.new(data) + .$promise.then(function (response) { + return response; + }) + .catch(function (response) { + vm.importingSa = false; + if (response.status == 409) { + storagePolicyExistsModal(data); + } + return response; + }); }) ); } else { @@ -205,6 +215,24 @@ export default class ImportCtrl { }); modalInstance.result.then(function (data) {}); } + function storagePolicyExistsModal(profile) { + const modalInstance = $uibModal.open({ + animation: true, + ariaLabelledBy: 'modal-title', + ariaDescribedBy: 'modal-body', + templateUrl: 'static/frontend/views/modals/storagePolicy-exists-modal.html', + controller: 'OverwriteModalInstanceCtrl', + controllerAs: '$ctrl', + resolve: { + data: function () { + return { + profile: profile, + }; + }, + }, + }); + modalInstance.result.then(function (data) {}); + } vm.triggerProfileUpload = function () { document.getElementById('profile-upload').click(); }; diff --git a/ESSArch_Core/frontend/static/frontend/scripts/controllers/OverwriteModalInstanceCtrl.js b/ESSArch_Core/frontend/static/frontend/scripts/controllers/OverwriteModalInstanceCtrl.js index ac0f11f77..478600089 100644 --- a/ESSArch_Core/frontend/static/frontend/scripts/controllers/OverwriteModalInstanceCtrl.js +++ b/ESSArch_Core/frontend/static/frontend/scripts/controllers/OverwriteModalInstanceCtrl.js @@ -1,5 +1,5 @@ export default class OverwriteModalInstanceCtrl { - constructor($uibModalInstance, data, Profile, SA, Notifications, $translate) { + constructor($uibModalInstance, data, Profile, SA, Notifications, $translate, StoragePolicy) { const $ctrl = this; if (data.file) { $ctrl.file = data.file; @@ -20,6 +20,18 @@ export default class OverwriteModalInstanceCtrl { return resource; }); }; + $ctrl.overwriteStoragePolicy = function () { + return StoragePolicy.update($ctrl.profile).$promise.then(function (resource) { + Notifications.add($translate.instant('IMPORT.STORAGEPOLICY_IMPORTED', resource), 'success', 5000, { + isHtml: true, + }); + $ctrl.data = { + status: 'overwritten', + }; + $uibModalInstance.close($ctrl.data); + return resource; + }); + }; $ctrl.overwriteSa = function () { return SA.update($ctrl.profile) .$promise.then(function (resource) { diff --git a/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.controllers.module.js b/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.controllers.module.js index 269918d70..ab12475ca 100644 --- a/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.controllers.module.js +++ b/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.controllers.module.js @@ -784,6 +784,7 @@ export default angular 'SA', 'Notifications', '$translate', + 'StoragePolicy', OverwriteModalInstanceCtrl, ]) .controller('PlaceNodeInArchiveModalInstanceCtrl', [ diff --git a/ESSArch_Core/frontend/static/frontend/scripts/services/storagePolicy.js b/ESSArch_Core/frontend/static/frontend/scripts/services/storagePolicy.js index 9c443885e..000544dd4 100644 --- a/ESSArch_Core/frontend/static/frontend/scripts/services/storagePolicy.js +++ b/ESSArch_Core/frontend/static/frontend/scripts/services/storagePolicy.js @@ -1,3 +1,19 @@ export default ($resource, appConfig) => { - return $resource(appConfig.djangoUrl + 'storage-policies/:id/:action/', {id: '@id'}, {}); + return $resource( + appConfig.djangoUrl + 'storage-policies/:id/:action/', + {}, + { + get: { + method: 'GET', + params: {id: '@id'}, + }, + new: { + method: 'POST', + }, + update: { + method: 'PUT', + params: {id: '@id'}, + }, + } + ); }; diff --git a/ESSArch_Core/frontend/static/frontend/views/modals/storagePolicy-exists-modal.html b/ESSArch_Core/frontend/static/frontend/views/modals/storagePolicy-exists-modal.html new file mode 100644 index 000000000..261d98c92 --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/views/modals/storagePolicy-exists-modal.html @@ -0,0 +1,19 @@ +
+ +
+
+
+

{{ "IMPORT.STORAGEPOLICY_EXISTS_DESC" | translate }}

+ {{ "NAME" | translate }}: {{ $ctrl.profile.policy_name }} +
+ ID: {{ $ctrl.profile.id }} +
+ Policy ID: {{ $ctrl.profile.policy_id }} +
+ AIS Policy ID: {{ $ctrl.profile.ais_project_id }} +
+
+ + +
+
diff --git a/ESSArch_Core/profiles/admin.py b/ESSArch_Core/profiles/admin.py index 3b74d887f..27629229e 100644 --- a/ESSArch_Core/profiles/admin.py +++ b/ESSArch_Core/profiles/admin.py @@ -57,7 +57,7 @@ def render_change_form(self, request, context, *args, **kwargs): fieldsets = ( (None, { 'classes': ('wide'), - 'fields': ('id', 'name', 'type', 'status', 'label', 'template',) + 'fields': ('id', 'name', 'type', 'status', 'label', 'policy', 'template',) }), ('Information about Archival organization', { 'classes': ('collapse', 'wide'), diff --git a/ESSArch_Core/profiles/models.py b/ESSArch_Core/profiles/models.py index 8435ce7bd..0ea8799aa 100644 --- a/ESSArch_Core/profiles/models.py +++ b/ESSArch_Core/profiles/models.py @@ -199,6 +199,7 @@ class SubmissionAgreement(models.Model): 'configuration.StoragePolicy', on_delete=models.PROTECT, related_name='submission_agreements', + verbose_name=_('storage policy'), ) include_profile_transfer_project = models.BooleanField(default=False) include_profile_content_type = models.BooleanField(default=False) From 6da00a11e9e8e0f916131808f278cb0da85c0465 Mon Sep 17 00:00:00 2001 From: henrikek Date: Thu, 4 Mar 2021 23:07:45 +0100 Subject: [PATCH 11/13] First try with remote storage --- ESSArch_Core/WorkflowEngine/models.py | 12 ++-- ESSArch_Core/WorkflowEngine/serializers.py | 5 +- ESSArch_Core/ip/serializers.py | 72 +------------------ .../migrations/0057_auto_20210304_1357.py | 21 ++++++ ESSArch_Core/storage/models.py | 2 +- 5 files changed, 36 insertions(+), 76 deletions(-) create mode 100644 ESSArch_Core/profiles/migrations/0057_auto_20210304_1357.py diff --git a/ESSArch_Core/WorkflowEngine/models.py b/ESSArch_Core/WorkflowEngine/models.py index 87785f502..df80085b1 100644 --- a/ESSArch_Core/WorkflowEngine/models.py +++ b/ESSArch_Core/WorkflowEngine/models.py @@ -604,15 +604,17 @@ def reraise(self): def create_remote_copy(self, session, host): create_remote_task_url = urljoin(host, reverse('processtask-list')) params = copy.deepcopy(self.params) - params.pop('_options', None) + params['storage_object'] = str(params['storage_object']) if params.get('storage_object') is not None else None ip_id = str(self.information_package.pk) if self.information_package.pk is not None else None + responsible_username = self.responsible.username if self.responsible is not None else None data = { 'id': str(self.pk), 'name': self.name, 'args': self.args, - 'params': self.params, + 'params': params, 'eager': self.eager, 'information_package': ip_id, + 'responsible': responsible_username, } r = session.post(create_remote_task_url, json=data, timeout=60) @@ -627,14 +629,16 @@ def create_remote_copy(self, session, host): def update_remote_copy(self, session, host): update_remote_task_url = urljoin(host, reverse('processtask-detail', args=(str(self.pk),))) params = copy.deepcopy(self.params) - params.pop('_options', None) + params['storage_object'] = str(params['storage_object']) if params.get('storage_object') is not None else None ip_id = str(self.information_package.pk) if self.information_package.pk is not None else None + responsible_username = self.responsible.username if self.responsible is not None else None data = { 'name': self.name, 'args': self.args, - 'params': self.params, + 'params': params, 'eager': self.eager, 'information_package': ip_id, + 'responsible': responsible_username, } r = session.patch(update_remote_task_url, json=data, timeout=60) diff --git a/ESSArch_Core/WorkflowEngine/serializers.py b/ESSArch_Core/WorkflowEngine/serializers.py index f7c9bd575..bab72524e 100644 --- a/ESSArch_Core/WorkflowEngine/serializers.py +++ b/ESSArch_Core/WorkflowEngine/serializers.py @@ -25,6 +25,7 @@ import uuid from celery import states as celery_states +from django.contrib.auth import get_user_model from rest_framework import serializers from ESSArch_Core.auth.fields import CurrentUsernameDefault @@ -33,6 +34,8 @@ from ESSArch_Core.WorkflowEngine.models import ProcessStep, ProcessTask from ESSArch_Core.WorkflowEngine.util import get_result +User = get_user_model() + class ProcessStepChildrenSerializer(serializers.Serializer): url = serializers.SerializerMethodField() @@ -84,7 +87,7 @@ class ProcessTaskSerializer(serializers.ModelSerializer): args = serializers.JSONField(required=False) params = serializers.SerializerMethodField() responsible = serializers.SlugRelatedField( - slug_field='username', read_only=True + slug_field='username', queryset=User.objects.all(), ) def get_params(self, obj): diff --git a/ESSArch_Core/ip/serializers.py b/ESSArch_Core/ip/serializers.py index 9666481ee..44724fb54 100644 --- a/ESSArch_Core/ip/serializers.py +++ b/ESSArch_Core/ip/serializers.py @@ -507,17 +507,11 @@ class Meta: class InformationPackageFromMasterSerializer(serializers.ModelSerializer): aic = InformationPackageAICSerializer(omit=['information_packages']) - policy = StoragePolicySerializer() - # organization = serializers.SerializerMethodField() + organization = serializers.SerializerMethodField() submission_agreement = serializers.PrimaryKeyRelatedField( queryset=SubmissionAgreement.objects.all(), pk_field=serializers.UUIDField(format='hex_verbose'), ) - # submission_agreement_data = serializers.SerializerMethodField() - # submission_agreement_data_versions = serializers.ListField( - # child=serializers.PrimaryKeyRelatedField(read_only=True) - # ) - # profiles = ProfileIPSerializer(source='profileip_set', many=True) def get_organization(self, obj): try: @@ -527,72 +521,11 @@ def get_organization(self, obj): except IndexError: return None - def get_submission_agreement_data(self, obj): - if obj.submission_agreement_data is not None: - serializer = SubmissionAgreementIPDataSerializer(obj.submission_agreement_data) - data = serializer.data - else: - data = {'data': {}} - - extra_data = fill_specification_data(ip=obj, sa=obj.submission_agreement) - - for field in getattr(obj.submission_agreement, 'template', []): - if field['key'] in extra_data: - data['data'][field['key']] = extra_data[field['key']] - - return data - - def create_storage_method(self, data): - if data is None: - return None - - storage_method_target_set_data = data.pop('storage_method_target_relations') - storage_method, _ = StorageMethod.objects.update_or_create( - id=data['id'], - defaults=data - ) - - for storage_method_target_data in storage_method_target_set_data: - storage_target_data = storage_method_target_data.pop('storage_target') - storage_target_data.pop('remote_server', None) - storage_target, _ = StorageTarget.objects.update_or_create( - id=storage_target_data['id'], - defaults=storage_target_data - ) - storage_method_target_data['storage_method'] = storage_method - storage_method_target_data['storage_target'] = storage_target - storage_method_target, _ = StorageMethodTargetRelation.objects.update_or_create( - id=storage_method_target_data['id'], - defaults=storage_method_target_data - ) - - return storage_method - def create(self, validated_data): aic_data = validated_data.pop('aic') aic_data['last_changed_local'] = timezone.now aic, _ = InformationPackage.objects.update_or_create(id=aic_data['id'], defaults=aic_data) - policy_data = validated_data.pop('policy') - storage_method_set_data = policy_data.pop('storage_methods') - - cache_storage_data = policy_data.pop('cache_storage') - ingest_path_data = policy_data.pop('ingest_path') - - cache_storage = self.create_storage_method(cache_storage_data) - ingest_path, _ = Path.objects.update_or_create(entity=ingest_path_data['entity'], defaults=ingest_path_data) - - policy_data['cache_storage'] = cache_storage - policy_data['ingest_path'] = ingest_path - - policy, _ = StoragePolicy.objects.update_or_create(policy_id=policy_data['policy_id'], - defaults=policy_data) - - for storage_method_data in storage_method_set_data: - storage_method = self.create_storage_method(storage_method_data) - policy.storage_methods.add(storage_method) - # add to policy, dummy - request = self.context.get("request") if request and hasattr(request, "user"): user = request.user @@ -600,7 +533,6 @@ def create(self, validated_data): user = User.objects.get(username="system") validated_data['aic'] = aic - # validated_data['policy'] = policy validated_data['responsible'] = user validated_data['last_changed_local'] = timezone.now ip, _ = InformationPackage.objects.update_or_create(id=validated_data['id'], defaults=validated_data) @@ -617,7 +549,7 @@ class Meta: 'message_digest', 'message_digest_algorithm', 'content_mets_create_date', 'content_mets_size', 'content_mets_digest_algorithm', 'content_mets_digest', 'package_mets_create_date', 'package_mets_size', 'package_mets_digest_algorithm', 'package_mets_digest', - 'start_date', 'end_date', 'appraisal_date', 'policy', 'submission_agreement', + 'start_date', 'end_date', 'appraisal_date', 'submission_agreement', 'organization', # 'start_date', 'end_date', 'appraisal_date', 'profiles', 'policy', 'organization', 'submission_agreement', # 'submission_agreement_locked', 'submission_agreement_data', 'submission_agreement_data_versions', ) diff --git a/ESSArch_Core/profiles/migrations/0057_auto_20210304_1357.py b/ESSArch_Core/profiles/migrations/0057_auto_20210304_1357.py new file mode 100644 index 000000000..3141d8ff0 --- /dev/null +++ b/ESSArch_Core/profiles/migrations/0057_auto_20210304_1357.py @@ -0,0 +1,21 @@ +# Generated by Django 3.1.2 on 2021-03-04 12:57 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('configuration', '0024_auto_20200309_1535'), + ('profiles', '0056_auto_20201014_1920'), + ] + + operations = [ + migrations.AlterField( + model_name='submissionagreement', + name='policy', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='submission_agreements', + to='configuration.storagepolicy', verbose_name='storage policy'), + ), + ] diff --git a/ESSArch_Core/storage/models.py b/ESSArch_Core/storage/models.py index 581d9aff5..523156f92 100644 --- a/ESSArch_Core/storage/models.py +++ b/ESSArch_Core/storage/models.py @@ -854,7 +854,7 @@ def read(self, dst, task, extract=False): # by master to write to its temp directory temp_dir = Path.objects.get(entity='temp').value - user, passw, host = storage_target.master_server.split(',') + host, user, passw = storage_target.master_server.split(',') session = requests.Session() session.verify = settings.REQUESTS_VERIFY session.auth = (user, passw) From 1faf555093ad6195e7a5d39169f79e7101ad01da Mon Sep 17 00:00:00 2001 From: henrikek Date: Wed, 10 Mar 2021 23:13:41 +0100 Subject: [PATCH 12/13] responsible field is not required for ProcesTaskSerializer --- ESSArch_Core/WorkflowEngine/serializers.py | 2 +- ESSArch_Core/ip/serializers.py | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/ESSArch_Core/WorkflowEngine/serializers.py b/ESSArch_Core/WorkflowEngine/serializers.py index bab72524e..201b13b48 100644 --- a/ESSArch_Core/WorkflowEngine/serializers.py +++ b/ESSArch_Core/WorkflowEngine/serializers.py @@ -87,7 +87,7 @@ class ProcessTaskSerializer(serializers.ModelSerializer): args = serializers.JSONField(required=False) params = serializers.SerializerMethodField() responsible = serializers.SlugRelatedField( - slug_field='username', queryset=User.objects.all(), + slug_field='username', queryset=User.objects.all(), required=False, ) def get_params(self, obj): diff --git a/ESSArch_Core/ip/serializers.py b/ESSArch_Core/ip/serializers.py index 44724fb54..39606f727 100644 --- a/ESSArch_Core/ip/serializers.py +++ b/ESSArch_Core/ip/serializers.py @@ -11,8 +11,7 @@ from ESSArch_Core.api.serializers import DynamicModelSerializer from ESSArch_Core.auth.fields import CurrentUsernameDefault from ESSArch_Core.auth.serializers import GroupSerializer, UserSerializer -from ESSArch_Core.configuration.models import EventType, Path, StoragePolicy -from ESSArch_Core.configuration.serializers import StoragePolicySerializer +from ESSArch_Core.configuration.models import EventType from ESSArch_Core.ip.models import ( Agent, AgentNote, @@ -32,11 +31,6 @@ SubmissionAgreementIPDataSerializer, ) from ESSArch_Core.profiles.utils import fill_specification_data, profile_types -from ESSArch_Core.storage.models import ( - StorageMethod, - StorageMethodTargetRelation, - StorageTarget, -) from ESSArch_Core.tags.models import ( Delivery, Structure, From 503087c45f31e9e5b76fccdd30ece73d6fd98f23 Mon Sep 17 00:00:00 2001 From: henrikek Date: Thu, 11 Mar 2021 19:40:04 +0100 Subject: [PATCH 13/13] Prevent to mark IP as preserved if local exception --- ESSArch_Core/ip/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ESSArch_Core/ip/models.py b/ESSArch_Core/ip/models.py index 862d8e85e..9ee55fee4 100644 --- a/ESSArch_Core/ip/models.py +++ b/ESSArch_Core/ip/models.py @@ -1768,6 +1768,8 @@ def preserve(self, src: list, storage_target, container: bool, task): task.result = remote_data['result'] task.traceback = remote_data['traceback'] task.exception = remote_data['exception'] + if task.status == 'SUCCESS': + storage_object = StorageObject.create_from_remote_copy(host, session, task.result) task.save() if task.status != celery_states.SUCCESS: @@ -1782,6 +1784,8 @@ def preserve(self, src: list, storage_target, container: bool, task): task.result = remote_data['result'] task.traceback = remote_data['traceback'] task.exception = remote_data['exception'] + if task.status == 'SUCCESS': + storage_object = StorageObject.create_from_remote_copy(host, session, task.result) task.save() sleep(5) @@ -1789,7 +1793,6 @@ def preserve(self, src: list, storage_target, container: bool, task): if task.status in celery_states.EXCEPTION_STATES: task.reraise() - storage_object = StorageObject.create_from_remote_copy(host, session, task.result) else: storage_medium, created = storage_target.get_or_create_storage_medium(qs=qs)