#710: Restore Entities, Purge Entities, ability to see deleted Entities

docs/api.yaml

@@ -40,14 +40,21 @@ info:
 
     Here major and breaking changes to the API are listed by version.
 
+    ## ODK Central v2025.1
+
+    **Added**:
+    - [RESTORE](/central-api-entity-management/#restoring-a-deleted-entity) endpoint for Entities.
+    - Entities that have been soft-deleted for 30 days will automatically be purged. 
+    - [Entities Odata](/central-api-odata-endpoints/#id3) now returns `__system/deletedAt`. It can also be used in $filter, $sort and $select query parameters.
+
     ## ODK Central v2024.3
 
     **Added**:
     - Endpoints for managing [User Preferences](/central-api-accounts-and-users/#user-preferences), mainly to be used by the frontend.
 
     **Changed**:
 
-    - [Submissions Odata]() now returns `__system/deletedAt`. It can also be used in $filter, $sort and $select query parameters.
+    - [Submissions Odata](/central-api-odata-endpoints/#data-document) now returns `__system/deletedAt`. It can also be used in $filter, $sort and $select query parameters.
     - Dataset (Entity List) properties with the same name but different capitalization are not allowed.
     - Form Attachments for both [published Forms](/central-api-form-management/#listing-form-attachments) and [draft Forms](/central-api-form-management/#listing-expected-draft-form-attachments) now return a property representing the hash of the attachment file.
 
@@ -9155,51 +9162,6 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error403'
-    delete:
-      tags:
-      - Entity Management
-      summary: Deleting an Entity
-      description: Use this API to delete an Entity. With this API, Entity is soft-deleted,
-        which means it is still in the database and you can retreive it by passing
-        `?deleted=true` to [GET /projects/:id/datasets/:name/entities](/central-api-entity-management/#entities-metadata).
-        In the future, we will provide a way to restore deleted entities and purge
-        deleted entities.
-      operationId: Deleting an Entity
-      parameters:
-      - name: projectId
-        in: path
-        description: The numeric ID of the Project
-        required: true
-        schema:
-          type: number
-        example: "16"
-      - name: name
-        in: path
-        description: Name of the Dataset
-        required: true
-        schema:
-          type: string
-        example: people
-      - name: uuid
-        in: path
-        description: UUID of the Entity
-        required: true
-        schema:
-          type: string
-        example: 54a405a0-53ce-4748-9788-d23a30cc3afa
-      responses:
-        200:
-          description: OK
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/Success'
-        403:
-          description: Forbidden
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/Error403'
     patch:
       tags:
       - Entity Management
@@ -9314,6 +9276,100 @@ paths:
               schema:
                 $ref: '#/components/schemas/Error403'
       x-codegen-request-body-name: body
+    delete:
+      tags:
+      - Entity Management
+      summary: Deleting an Entity
+      description: Use this API to delete an Entity. With this API, Entity is soft-deleted,
+        which means it is still in the database and you can retreive it by passing
+        `?deleted=true` to [GET /projects/:id/datasets/:name/entities](/central-api-entity-management/#entities-metadata).
+        In the future, we will provide a way to restore deleted entities and purge
+        deleted entities.
+      operationId: Deleting an Entity
+      parameters:
+      - name: projectId
+        in: path
+        description: The numeric ID of the Project
+        required: true
+        schema:
+          type: number
+        example: "16"
+      - name: name
+        in: path
+        description: Name of the Dataset
+        required: true
+        schema:
+          type: string
+        example: people
+      - name: uuid
+        in: path
+        description: UUID of the Entity
+        required: true
+        schema:
+          type: string
+        example: 54a405a0-53ce-4748-9788-d23a30cc3afa
+      responses:
+        200:
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Success'
+        403:
+          description: Forbidden
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error403'
+
+  /projects/{projectId}/datasets/{name}/entities/{uuid}/restore:
+    post:
+      tags:
+      - Entity Management
+      summary: Restoring a deleted Entity
+      description: Entities that have been recently soft-deleted and not yet purged can be restored using this endpoint.
+      operationId: Restoring a deleted Entity
+      parameters:
+      - name: projectId
+        in: path
+        description: The numeric ID of the Project
+        required: true
+        schema:
+          type: number
+        example: "16"
+      - name: name
+        in: path
+        description: Name of the Dataset
+        required: true
+        schema:
+          type: string
+        example: people
+      - name: uuid
+        in: path
+        description: UUID of the Entity
+        required: true
+        schema:
+          type: string
+        example: 54a405a0-53ce-4748-9788-d23a30cc3afa
+      responses:
+        200:
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Success'
+        403:
+          description: Forbidden
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error403'
+        404:
+          description: Not Found
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error404'
   /projects/{projectId}/datasets/{name}/entities/{uuid}/versions:
     get:
       tags:
@@ -10912,8 +10968,9 @@ paths:
         | Entity Timestamp        | `__system/createdAt` |
         | Entity Update Timestamp | `__system/updatedAt` |
         | Entity Conflict         | `__system/conflict`  |
+        | Entity Delete Timestamp | `__system/deletedAt` |
 
-        Note that `createdAt` and `updatedAt` are time components. This means that any comparisons you make need to account for the full time of the entity. It might seem like `$filter=__system/createdAt le 2020-01-31` would return all results on or before 31 Jan 2020, but in fact only entities made before midnight of that day would be accepted. To include all of the month of January, you need to filter by either `$filter=__system/createdAt le 2020-01-31T23:59:59.999Z` or `$filter=__system/createdAt lt 2020-02-01`. Remember also that you can [query by a specific timezone](https://en.wikipedia.org/wiki/ISO_8601#Time_offsets_from_UTC).
+        Note that `createdAt`, `updatedAt` and `deletedAt` are time components. This means that any comparisons you make need to account for the full time of the entity. It might seem like `$filter=__system/createdAt le 2020-01-31` would return all results on or before 31 Jan 2020, but in fact only entities made before midnight of that day would be accepted. To include all of the month of January, you need to filter by either `$filter=__system/createdAt le 2020-01-31T23:59:59.999Z` or `$filter=__system/createdAt lt 2020-02-01`. Remember also that you can [query by a specific timezone](https://en.wikipedia.org/wiki/ISO_8601#Time_offsets_from_UTC).
 
         Please see the [OData documentation](http://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#_Toc31358948) on `$filter` [operations](http://docs.oasis-open.org/odata/odata/v4.01/cs01/part1-protocol/odata-v4.01-cs01-part1-protocol.html#sec_BuiltinFilterOperations) and [functions](http://docs.oasis-open.org/odata/odata/v4.01/cs01/part1-protocol/odata-v4.01-cs01-part1-protocol.html#sec_BuiltinQueryFunctions) for more information.
 
@@ -11007,6 +11064,7 @@ paths:
                     creatorName: Tree surveyor
                     updates: 1,
                     updatedAt: '2023-04-31T19:41:16.478Z'
+                    deletedAt: null
                     version: 1
                     conflict: null
                 - __id: aeebd746-3b1e-4a24-ba9d-ed6547bd5ff1
@@ -11017,6 +11075,7 @@ paths:
                     creatorName: Tree surveyor
                     updates: 1,
                     updatedAt: '2023-04-31T19:41:16.478Z'
+                    deletedAt: null
                     version: 2
                     conflict: null
                   geometry: 47.722581 18.562111 0 0,

lib/bin/purge.js

@@ -21,11 +21,13 @@ const { purgeTask } = require('../task/purge');
 
 const { program } = require('commander');
 program.option('-f, --force', 'Force any soft-deleted form to be purged right away.');
-program.option('-m, --mode <value>', 'Mode of purging. Can be "forms", "submissions", or "all". Default is "all".', 'all');
+program.option('-m, --mode <value>', 'Mode of purging. Can be "forms", "submissions", "entities" or "all". Default is "all".', 'all');
 program.option('-i, --formId <integer>', 'Purge a specific form based on its id.', parseInt);
 program.option('-p, --projectId <integer>', 'Restrict purging to a specific project.', parseInt);
 program.option('-x, --xmlFormId <value>', 'Restrict purging to specific form based on xmlFormId (must be used with projectId).');
 program.option('-s, --instanceId <value>', 'Restrict purging to a specific submission based on instanceId (use with projectId and xmlFormId).');
+program.option('-d, --datasetName <value>', 'Restrict purging to specific dataset/entity-list based on its name (must be used with projectId).');
+program.option('-e, --entityUuid <value>', 'Restrict purging to a specific entitiy based on its UUID (use with projectId and datasetName).');
 
 program.parse();
 

lib/data/entity.js

@@ -24,6 +24,7 @@ const { sanitizeOdataIdentifier, blankStringToNull, isBlank } = require('../util
 const odataToColumnMap = new Map([
   ['__system/createdAt', 'entities.createdAt'],
   ['__system/updatedAt', 'entities.updatedAt'],
+  ['__system/deletedAt', 'entities.deletedAt'],
   ['__system/creatorId', 'entities.creatorId'],
   ['__system/conflict', 'entities.conflict']
 ]);
@@ -284,6 +285,7 @@ const selectFields = (entity, properties, selectedProperties) => {
     creatorName: entity.aux.creator.displayName,
     updates: entity.updates,
     updatedAt: entity.updatedAt,
+    deletedAt: entity.deletedAt,
     version: entity.def.version,
     conflict: entity.conflict
   };

lib/formats/odata.js

@@ -233,6 +233,7 @@ const edmxTemplaterForEntities = template(`<?xml version="1.0" encoding="UTF-8"?
         <Property Name="creatorName" Type="Edm.String"/>        
         <Property Name="updates" Type="Edm.Int64"/>
         <Property Name="updatedAt" Type="Edm.DateTimeOffset"/>
+        <Property Name="deletedAt" Type="Edm.DateTimeOffset"/>
         <Property Name="version" Type="Edm.Int64"/>
         <Property Name="conflict" Type="Edm.String"/>
       </ComplexType>

lib/model/migrations/20241224-01-entity-restore-verb.js

@@ -0,0 +1,20 @@
+// Copyright 2024 ODK Central Developers
+// See the NOTICE file at the top-level directory of this distribution and at
+// https://github.com/getodk/central-backend/blob/master/NOTICE.
+// This file is part of ODK Central. It is subject to the license terms in
+// the LICENSE file found in the top-level directory of this distribution and at
+// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central,
+// including this file, may be copied, modified, propagated, or distributed
+// except according to the terms contained in the LICENSE file.
+
+const up = (db) => db.raw(`
+UPDATE roles
+SET verbs = verbs || '["entity.restore"]'::jsonb
+WHERE system IN ('admin', 'manager')`);
+
+const down = (db) => db.raw(`
+UPDATE roles
+SET verbs = verbs - 'entity.restore'
+WHERE system IN ('admin', 'manager')`);
+
+module.exports = { up, down };

lib/model/migrations/20241224-02-cascade-entity-purge.js

@@ -0,0 +1,20 @@
+// Copyright 2024 ODK Central Developers
+// See the NOTICE file at the top-level directory of this distribution and at
+// https://github.com/getodk/central-backend/blob/master/NOTICE.
+// This file is part of ODK Central. It is subject to the license terms in
+// the LICENSE file found in the top-level directory of this distribution and at
+// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central,
+// including this file, may be copied, modified, propagated, or distributed
+// except according to the terms contained in the LICENSE file.
+
+const up = (db) => db.raw(`
+ALTER TABLE public.entity_defs DROP CONSTRAINT entity_defs_entityid_foreign;
+ALTER TABLE public.entity_defs ADD CONSTRAINT entity_defs_entityid_foreign FOREIGN KEY ("entityId") REFERENCES public.entities(id) ON DELETE CASCADE;
+`);
+
+const down = ((db) => db.raw(`
+ALTER TABLE public.entity_defs DROP CONSTRAINT entity_defs_entityid_foreign;
+ALTER TABLE public.entity_defs ADD CONSTRAINT entity_defs_entityid_foreign FOREIGN KEY ("entityId") REFERENCES public.entities(id);
+`));
+
+module.exports = { up, down };

lib/model/migrations/20241226-01-indices-for-purging-entities.js

@@ -0,0 +1,24 @@
+// Copyright 2024 ODK Central Developers
+// See the NOTICE file at the top-level directory of this distribution and at
+// https://github.com/getodk/central-backend/blob/master/NOTICE.
+// This file is part of ODK Central. It is subject to the license terms in
+// the LICENSE file found in the top-level directory of this distribution and at
+// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central,
+// including this file, may be copied, modified, propagated, or distributed
+// except according to the terms contained in the LICENSE file.
+
+const up = (db) => db.raw(`
+CREATE INDEX audits_details_entity_uuid ON public.audits USING hash ((details->'entity'->>'uuid'))
+WHERE ACTION IN ('entity.create', 'entity.update', 'entity.update.version', 'entity.update.resolve', 'entity.delete', 'entity.restore');
+
+CREATE INDEX audits_details_entityUuids ON audits USING gin ((details -> 'entityUuids') jsonb_path_ops)
+WHERE ACTION = 'entities.purge';
+`);
+
+const down = ((db) => db.raw(`
+DROP INDEX audits_details_entity_uuid;
+DROP INDEX audits_details_entityUuids;
+`));
+
+module.exports = { up, down };
+

lib/model/migrations/20241227-01-backfill-audit-entity-uuid.js

@@ -0,0 +1,17 @@
+// Copyright 2024 ODK Central Developers
+// See the NOTICE file at the top-level directory of this distribution and at
+// https://github.com/getodk/central-backend/blob/master/NOTICE.
+// This file is part of ODK Central. It is subject to the license terms in
+// the LICENSE file found in the top-level directory of this distribution and at
+// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central,
+// including this file, may be copied, modified, propagated, or distributed
+// except according to the terms contained in the LICENSE file.
+
+const up = (db) => db.raw(`
+UPDATE audits SET details = ('{ "entity":{ "uuid":"' || (details->>'uuid') || '"}}')::JSONB
+WHERE action = 'entity.delete' AND details \\? 'uuid';
+`);
+
+const down = () => {};
+
+module.exports = { up, down };

lib/model/query/audits.js

@@ -150,11 +150,19 @@ const getBySubmissionId = (submissionId, options) => ({ all }) => _getBySubmissi
 // There is a separate query below to assemble full submission details for non-deleted
 // submissions, but it was far too slow to have be part of this query.
 const _getByEntityId = (fields, options, entityId) => sql`
-SELECT ${fields} FROM entity_defs
-
-  LEFT JOIN entity_def_sources on entity_def_sources.id = entity_defs."sourceId"
-  INNER JOIN audits ON ((audits.details->>'entityDefId')::INTEGER = entity_defs.id OR (audits.details->>'sourceId')::INTEGER = entity_def_sources.id)
-
+SELECT ${fields} FROM (
+    SELECT audits.* FROM audits 
+    JOIN entities ON (audits.details->'entity'->>'uuid') = entities.uuid
+    WHERE entities.id = ${entityId}
+    UNION ALL
+    SELECT audits.* FROM audits
+    JOIN entity_def_sources ON (audits.details->>'sourceId')::INTEGER = entity_def_sources.id
+    JOIN entity_defs ON entity_def_sources.id = entity_defs."sourceId"
+    WHERE entity_defs."entityId" = ${entityId} AND audits.action = 'entity.bulk.create'
+  ) audits
+
+  LEFT JOIN entity_defs ON (audits.details->>'entityDefId')::INTEGER = entity_defs.id
+  LEFT JOIN entity_def_sources ON entity_defs."sourceId" = entity_def_sources.id OR (audits.details->>'sourceId')::INTEGER = entity_def_sources.id
   LEFT JOIN actors ON actors.id=audits."actorId"
 
   LEFT JOIN audits triggering_event ON entity_def_sources."auditId" = triggering_event.id
@@ -165,7 +173,6 @@ SELECT ${fields} FROM entity_defs
   LEFT JOIN audits submission_create_event ON (triggering_event.details->'submissionId')::INTEGER = (submission_create_event.details->'submissionId')::INTEGER AND submission_create_event.action = 'submission.create'
   LEFT JOIN actors submission_create_event_actor ON submission_create_event_actor.id = submission_create_event."actorId"
 
-  WHERE entity_defs."entityId" = ${entityId}
   ORDER BY audits."loggedAt" DESC, audits.id DESC
     ${page(options)}`;
 
@@ -253,13 +260,17 @@ const getByEntityId = (entityId, options) => ({ all }) => {
 
         // Look up the full Submission information and attempt to merge it in if it exists.
         const subOption = entityDefDict[audit.aux.def.id];
-        const fullSubmission = subOption.aux.submission
-          .map(s => s.withAux('submitter', subOption.aux.submissionActor.orNull()))
-          .map(s => s.withAux('currentVersion', subOption.aux.currentVersion.map(v => v.withAux('submitter', subOption.aux.currentSubmissionActor.orNull()))))
-          .map(s => s.forApi())
-          .map(s => mergeLeft(s, { xmlFormId: subOption.aux.form.map(f => f.xmlFormId).orNull() }));
+        let submission;
+
+        if (subOption) {
+          const fullSubmission = subOption.aux.submission
+            .map(s => s.withAux('submitter', subOption.aux.submissionActor.orNull()))
+            .map(s => s.withAux('currentVersion', subOption.aux.currentVersion.map(v => v.withAux('submitter', subOption.aux.currentSubmissionActor.orNull()))))
+            .map(s => s.forApi())
+            .map(s => mergeLeft(s, { xmlFormId: subOption.aux.form.map(f => f.xmlFormId).orNull() }));
 
-        const submission = mergeLeft(baseSubmission, fullSubmission.orElse(undefined));
+          submission = mergeLeft(baseSubmission, fullSubmission.orElse(undefined));
+        }
 
         // Note: The source added to each audit event represents the source of the
         // corresponding entity _version_, rather than the source of the event.