Use ResourceListLake correctly

microsoft · Dec 4, 2024 · 9992090 · 9992090
1 parent 8b660a0
commit 9992090
Show file tree

Hide file tree

Showing 4 changed files with 59 additions and 25 deletions.
diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/84.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/84.diff.sql
@@ -582,9 +582,9 @@ END
   COMMIT TRANSACTION
 END
 GO
-GO
 BEGIN TRANSACTION
 GO
+DROP PROCEDURE CaptureResourceIdsForChanges
 DROP PROCEDURE MergeResources
 DROP PROCEDURE UpdateResourceSearchParams
 DROP TYPE ReferenceSearchParamList
@@ -602,6 +602,16 @@ CREATE TYPE dbo.ReferenceSearchParamList AS TABLE
    UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId) 
 )
 GO
+CREATE PROCEDURE dbo.CaptureResourceIdsForChanges @Resources dbo.ResourceList READONLY, @ResourcesLake dbo.ResourceListLake READONLY
+AS
+set nocount on
+-- This procedure is intended to be called from the MergeResources procedure and relies on its transaction logic
+INSERT INTO dbo.ResourceChangeData 
+       ( ResourceId, ResourceTypeId, ResourceVersion,                                              ResourceChangeTypeId )
+  SELECT ResourceId, ResourceTypeId,         Version, CASE WHEN IsDeleted = 1 THEN 2 WHEN Version > 1 THEN 1 ELSE 0 END
+    FROM (SELECT ResourceId, ResourceTypeId, Version, IsHistory, IsDeleted FROM @Resources UNION ALL SELECT ResourceId, ResourceTypeId, Version, IsHistory, IsDeleted FROM @ResourcesLake) A
+    WHERE IsHistory = 0
+GO
 CREATE PROCEDURE dbo.UpdateResourceSearchParams
     @FailedResources int = 0 OUT
    ,@Resources dbo.ResourceList READONLY
@@ -794,7 +804,7 @@ DECLARE @st datetime = getUTCdate()
        ,@NewIdsCount int
        ,@FirstIdInt bigint
 
-DECLARE @Mode varchar(200) = isnull((SELECT 'RT=['+convert(varchar,min(ResourceTypeId))+','+convert(varchar,max(ResourceTypeId))+'] Sur=['+convert(varchar,min(ResourceSurrogateId))+','+convert(varchar,max(ResourceSurrogateId))+'] V='+convert(varchar,max(Version))+' Rows='+convert(varchar,count(*)) FROM @Resources),'Input=Empty')
+DECLARE @Mode varchar(200) = isnull((SELECT 'RT=['+convert(varchar,min(ResourceTypeId))+','+convert(varchar,max(ResourceTypeId))+'] Sur=['+convert(varchar,min(ResourceSurrogateId))+','+convert(varchar,max(ResourceSurrogateId))+'] V='+convert(varchar,max(Version))+' Rows='+convert(varchar,count(*)) FROM (SELECT ResourceTypeId, ResourceSurrogateId, Version FROM @Resources UNION ALL SELECT ResourceTypeId, ResourceSurrogateId, Version FROM @ResourcesLake) A),'Input=Empty')
 SET @Mode += ' E='+convert(varchar,@RaiseExceptionOnConflict)+' CC='+convert(varchar,@IsResourceChangeCaptureEnabled)+' IT='+convert(varchar,@InitialTranCount)+' T='+isnull(convert(varchar,@TransactionId),'NULL')
 
 SET @AffectedRows = 0
@@ -810,6 +820,7 @@ BEGIN TRY
     (
         ResourceTypeId       smallint            NOT NULL
        ,ResourceSurrogateId  bigint              NOT NULL
+       ,ResourceId           varchar(64)         COLLATE Latin1_General_100_CS_AS NOT NULL
        ,ResourceIdInt        bigint              NOT NULL
        ,Version              int                 NOT NULL
        ,HasVersionToCompare  bit                 NOT NULL -- in case of multiple versions per resource indicates that row contains (existing version + 1) value
@@ -931,8 +942,8 @@ BEGIN TRY
   END
 
   INSERT INTO @ResourcesWithIds
-         (   ResourceTypeId,                           ResourceIdInt, Version, HasVersionToCompare, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, KeepHistory, RawResource, IsRawResourceMetaSet, SearchParamHash, FileId, OffsetInFile )
-    SELECT A.ResourceTypeId, isnull(C.ResourceIdInt,B.ResourceIdInt), Version, HasVersionToCompare, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, KeepHistory, RawResource, IsRawResourceMetaSet, SearchParamHash, FileId, OffsetInFile
+         (   ResourceTypeId,   ResourceId,                           ResourceIdInt, Version, HasVersionToCompare, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, KeepHistory, RawResource, IsRawResourceMetaSet, SearchParamHash, FileId, OffsetInFile )
+    SELECT A.ResourceTypeId, A.ResourceId, isnull(C.ResourceIdInt,B.ResourceIdInt), Version, HasVersionToCompare, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, KeepHistory, RawResource, IsRawResourceMetaSet, SearchParamHash, FileId, OffsetInFile
       FROM (SELECT ResourceTypeId, ResourceId, Version, HasVersionToCompare, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, KeepHistory, RawResource, IsRawResourceMetaSet, SearchParamHash, FileId = CASE WHEN OffsetInFile IS NOT NULL THEN @TransactionId END, OffsetInFile FROM @ResourcesLake
             UNION ALL
             SELECT ResourceTypeId, ResourceId, Version, HasVersionToCompare, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, KeepHistory, RawResource, IsRawResourceMetaSet, SearchParamHash,                                                                NULL,         NULL FROM @Resources
@@ -992,7 +1003,7 @@ BEGIN TRY
   IF @InitialTranCount = 0
   BEGIN
     IF EXISTS (SELECT * -- This extra statement avoids putting range locks when we don't need them
-                 FROM @Resources A JOIN dbo.Resource B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+                 FROM @ResourcesWithIds A JOIN dbo.Resource B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
                  WHERE B.IsHistory = 0
               )
     BEGIN
@@ -1001,14 +1012,14 @@ BEGIN TRY
       INSERT INTO @Existing
               (  ResourceTypeId,           SurrogateId )
         SELECT B.ResourceTypeId, B.ResourceSurrogateId
-          FROM (SELECT TOP (@DummyTop) * FROM @Resources) A
+          FROM (SELECT TOP (@DummyTop) * FROM @ResourcesWithIds) A
                JOIN dbo.Resource B WITH (ROWLOCK, HOLDLOCK) ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
           WHERE B.IsHistory = 0
             AND B.ResourceId = A.ResourceId
             AND B.Version = A.Version
           OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
 
-      IF @@rowcount = (SELECT count(*) FROM @Resources) SET @IsRetry = 1
+      IF @@rowcount = (SELECT count(*) FROM @ResourcesWithIds) SET @IsRetry = 1
 
       IF @IsRetry = 0 COMMIT TRANSACTION -- commit check transaction 
     END
@@ -1339,7 +1350,7 @@ BEGIN TRY
   END
 
   IF @IsResourceChangeCaptureEnabled = 1 --If the resource change capture feature is enabled, to execute a stored procedure called CaptureResourceChanges to insert resource change data.
-    EXECUTE dbo.CaptureResourceIdsForChanges @Resources
+    EXECUTE dbo.CaptureResourceIdsForChanges @Resources, @ResourcesLake
 
   IF @TransactionId IS NOT NULL
     EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId

diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/84.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/84.sql
@@ -1536,15 +1536,27 @@ END
 
 GO
 CREATE PROCEDURE dbo.CaptureResourceIdsForChanges
-@Resources dbo.ResourceList READONLY
+@Resources dbo.ResourceList READONLY, @ResourcesLake dbo.ResourceListLake READONLY
 AS
 SET NOCOUNT ON;
 INSERT INTO dbo.ResourceChangeData (ResourceId, ResourceTypeId, ResourceVersion, ResourceChangeTypeId)
 SELECT ResourceId,
        ResourceTypeId,
        Version,
        CASE WHEN IsDeleted = 1 THEN 2 WHEN Version > 1 THEN 1 ELSE 0 END
-FROM   @Resources
+FROM   (SELECT ResourceId,
+               ResourceTypeId,
+               Version,
+               IsHistory,
+               IsDeleted
+        FROM   @Resources
+        UNION ALL
+        SELECT ResourceId,
+               ResourceTypeId,
+               Version,
+               IsHistory,
+               IsDeleted
+        FROM   @ResourcesLake) AS A
 WHERE  IsHistory = 0;
 
 GO
@@ -3648,7 +3660,15 @@ AS
 SET NOCOUNT ON;
 DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = object_name(@@procid), @DummyTop AS BIGINT = 9223372036854775807, @InitialTranCount AS INT = @@trancount, @IsRetry AS BIT = 0, @RT AS SMALLINT, @NewIdsCount AS INT, @FirstIdInt AS BIGINT;
 DECLARE @Mode AS VARCHAR (200) = isnull((SELECT 'RT=[' + CONVERT (VARCHAR, min(ResourceTypeId)) + ',' + CONVERT (VARCHAR, max(ResourceTypeId)) + '] Sur=[' + CONVERT (VARCHAR, min(ResourceSurrogateId)) + ',' + CONVERT (VARCHAR, max(ResourceSurrogateId)) + '] V=' + CONVERT (VARCHAR, max(Version)) + ' Rows=' + CONVERT (VARCHAR, count(*))
-                                         FROM   @Resources), 'Input=Empty');
+                                         FROM   (SELECT ResourceTypeId,
+                                                        ResourceSurrogateId,
+                                                        Version
+                                                 FROM   @Resources
+                                                 UNION ALL
+                                                 SELECT ResourceTypeId,
+                                                        ResourceSurrogateId,
+                                                        Version
+                                                 FROM   @ResourcesLake) AS A), 'Input=Empty');
 SET @Mode += ' E=' + CONVERT (VARCHAR, @RaiseExceptionOnConflict) + ' CC=' + CONVERT (VARCHAR, @IsResourceChangeCaptureEnabled) + ' IT=' + CONVERT (VARCHAR, @InitialTranCount) + ' T=' + isnull(CONVERT (VARCHAR, @TransactionId), 'NULL');
 SET @AffectedRows = 0;
 RetryResourceIdIntMapInsert:
@@ -3672,6 +3692,7 @@ BEGIN TRY
     DECLARE @ResourcesWithIds AS TABLE (
         ResourceTypeId       SMALLINT        NOT NULL,
         ResourceSurrogateId  BIGINT          NOT NULL,
+        ResourceId           VARCHAR (64)    COLLATE Latin1_General_100_CS_AS NOT NULL,
         ResourceIdInt        BIGINT          NOT NULL,
         Version              INT             NOT NULL,
         HasVersionToCompare  BIT             NOT NULL,
@@ -3820,8 +3841,9 @@ BEGIN TRY
             DELETE @RTs
             WHERE  ResourceTypeId = @RT;
         END
-    INSERT INTO @ResourcesWithIds (ResourceTypeId, ResourceIdInt, Version, HasVersionToCompare, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, KeepHistory, RawResource, IsRawResourceMetaSet, SearchParamHash, FileId, OffsetInFile)
+    INSERT INTO @ResourcesWithIds (ResourceTypeId, ResourceId, ResourceIdInt, Version, HasVersionToCompare, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, KeepHistory, RawResource, IsRawResourceMetaSet, SearchParamHash, FileId, OffsetInFile)
     SELECT A.ResourceTypeId,
+           A.ResourceId,
            isnull(C.ResourceIdInt, B.ResourceIdInt),
            Version,
            HasVersionToCompare,
@@ -3920,7 +3942,7 @@ BEGIN TRY
     IF @InitialTranCount = 0
         BEGIN
             IF EXISTS (SELECT *
-                       FROM   @Resources AS A
+                       FROM   @ResourcesWithIds AS A
                               INNER JOIN
                               dbo.Resource AS B
                               ON B.ResourceTypeId = A.ResourceTypeId
@@ -3932,7 +3954,7 @@ BEGIN TRY
                     SELECT B.ResourceTypeId,
                            B.ResourceSurrogateId
                     FROM   (SELECT TOP (@DummyTop) *
-                            FROM   @Resources) AS A
+                            FROM   @ResourcesWithIds) AS A
                            INNER JOIN
                            dbo.Resource AS B WITH (ROWLOCK, HOLDLOCK)
                            ON B.ResourceTypeId = A.ResourceTypeId
@@ -3942,7 +3964,7 @@ BEGIN TRY
                            AND B.Version = A.Version
                     OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1));
                     IF @@rowcount = (SELECT count(*)
-                                     FROM   @Resources)
+                                     FROM   @ResourcesWithIds)
                         SET @IsRetry = 1;
                     IF @IsRetry = 0
                         COMMIT TRANSACTION;
@@ -4615,7 +4637,7 @@ BEGIN TRY
             SET @AffectedRows += @@rowcount;
         END
     IF @IsResourceChangeCaptureEnabled = 1
-        EXECUTE dbo.CaptureResourceIdsForChanges @Resources;
+        EXECUTE dbo.CaptureResourceIdsForChanges @Resources, @ResourcesLake;
     IF @TransactionId IS NOT NULL
         EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId;
     IF @InitialTranCount = 0

diff --git a/...crosoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/CaptureResourceIdsForChanges.sql b/...crosoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/CaptureResourceIdsForChanges.sql
@@ -1,10 +1,10 @@
-CREATE PROCEDURE dbo.CaptureResourceIdsForChanges @Resources dbo.ResourceList READONLY
+CREATE PROCEDURE dbo.CaptureResourceIdsForChanges @Resources dbo.ResourceList READONLY, @ResourcesLake dbo.ResourceListLake READONLY
 AS
 set nocount on
 -- This procedure is intended to be called from the MergeResources procedure and relies on its transaction logic
 INSERT INTO dbo.ResourceChangeData 
        ( ResourceId, ResourceTypeId, ResourceVersion,                                              ResourceChangeTypeId )
   SELECT ResourceId, ResourceTypeId,         Version, CASE WHEN IsDeleted = 1 THEN 2 WHEN Version > 1 THEN 1 ELSE 0 END
-    FROM @Resources
+    FROM (SELECT ResourceId, ResourceTypeId, Version, IsHistory, IsDeleted FROM @Resources UNION ALL SELECT ResourceId, ResourceTypeId, Version, IsHistory, IsDeleted FROM @ResourcesLake) A
     WHERE IsHistory = 0
 GO
diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResources.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeResources.sql
@@ -38,7 +38,7 @@ DECLARE @st datetime = getUTCdate()
        ,@NewIdsCount int
        ,@FirstIdInt bigint
 
-DECLARE @Mode varchar(200) = isnull((SELECT 'RT=['+convert(varchar,min(ResourceTypeId))+','+convert(varchar,max(ResourceTypeId))+'] Sur=['+convert(varchar,min(ResourceSurrogateId))+','+convert(varchar,max(ResourceSurrogateId))+'] V='+convert(varchar,max(Version))+' Rows='+convert(varchar,count(*)) FROM @Resources),'Input=Empty')
+DECLARE @Mode varchar(200) = isnull((SELECT 'RT=['+convert(varchar,min(ResourceTypeId))+','+convert(varchar,max(ResourceTypeId))+'] Sur=['+convert(varchar,min(ResourceSurrogateId))+','+convert(varchar,max(ResourceSurrogateId))+'] V='+convert(varchar,max(Version))+' Rows='+convert(varchar,count(*)) FROM (SELECT ResourceTypeId, ResourceSurrogateId, Version FROM @Resources UNION ALL SELECT ResourceTypeId, ResourceSurrogateId, Version FROM @ResourcesLake) A),'Input=Empty')
 SET @Mode += ' E='+convert(varchar,@RaiseExceptionOnConflict)+' CC='+convert(varchar,@IsResourceChangeCaptureEnabled)+' IT='+convert(varchar,@InitialTranCount)+' T='+isnull(convert(varchar,@TransactionId),'NULL')
 
 SET @AffectedRows = 0
@@ -54,6 +54,7 @@ BEGIN TRY
     (
         ResourceTypeId       smallint            NOT NULL
        ,ResourceSurrogateId  bigint              NOT NULL
+       ,ResourceId           varchar(64)         COLLATE Latin1_General_100_CS_AS NOT NULL
        ,ResourceIdInt        bigint              NOT NULL
        ,Version              int                 NOT NULL
        ,HasVersionToCompare  bit                 NOT NULL -- in case of multiple versions per resource indicates that row contains (existing version + 1) value
@@ -175,8 +176,8 @@ BEGIN TRY
   END
 
   INSERT INTO @ResourcesWithIds
-         (   ResourceTypeId,                           ResourceIdInt, Version, HasVersionToCompare, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, KeepHistory, RawResource, IsRawResourceMetaSet, SearchParamHash, FileId, OffsetInFile )
-    SELECT A.ResourceTypeId, isnull(C.ResourceIdInt,B.ResourceIdInt), Version, HasVersionToCompare, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, KeepHistory, RawResource, IsRawResourceMetaSet, SearchParamHash, FileId, OffsetInFile
+         (   ResourceTypeId,   ResourceId,                           ResourceIdInt, Version, HasVersionToCompare, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, KeepHistory, RawResource, IsRawResourceMetaSet, SearchParamHash, FileId, OffsetInFile )
+    SELECT A.ResourceTypeId, A.ResourceId, isnull(C.ResourceIdInt,B.ResourceIdInt), Version, HasVersionToCompare, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, KeepHistory, RawResource, IsRawResourceMetaSet, SearchParamHash, FileId, OffsetInFile
       FROM (SELECT ResourceTypeId, ResourceId, Version, HasVersionToCompare, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, KeepHistory, RawResource, IsRawResourceMetaSet, SearchParamHash, FileId = CASE WHEN OffsetInFile IS NOT NULL THEN @TransactionId END, OffsetInFile FROM @ResourcesLake
             UNION ALL
             SELECT ResourceTypeId, ResourceId, Version, HasVersionToCompare, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, KeepHistory, RawResource, IsRawResourceMetaSet, SearchParamHash,                                                                NULL,         NULL FROM @Resources
@@ -236,7 +237,7 @@ BEGIN TRY
   IF @InitialTranCount = 0
   BEGIN
     IF EXISTS (SELECT * -- This extra statement avoids putting range locks when we don't need them
-                 FROM @Resources A JOIN dbo.Resource B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+                 FROM @ResourcesWithIds A JOIN dbo.Resource B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
                  WHERE B.IsHistory = 0
               )
     BEGIN
@@ -245,14 +246,14 @@ BEGIN TRY
       INSERT INTO @Existing
               (  ResourceTypeId,           SurrogateId )
         SELECT B.ResourceTypeId, B.ResourceSurrogateId
-          FROM (SELECT TOP (@DummyTop) * FROM @Resources) A
+          FROM (SELECT TOP (@DummyTop) * FROM @ResourcesWithIds) A
                JOIN dbo.Resource B WITH (ROWLOCK, HOLDLOCK) ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
           WHERE B.IsHistory = 0
             AND B.ResourceId = A.ResourceId
             AND B.Version = A.Version
           OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
 
-      IF @@rowcount = (SELECT count(*) FROM @Resources) SET @IsRetry = 1
+      IF @@rowcount = (SELECT count(*) FROM @ResourcesWithIds) SET @IsRetry = 1
 
       IF @IsRetry = 0 COMMIT TRANSACTION -- commit check transaction 
     END
@@ -583,7 +584,7 @@ BEGIN TRY
   END
 
   IF @IsResourceChangeCaptureEnabled = 1 --If the resource change capture feature is enabled, to execute a stored procedure called CaptureResourceChanges to insert resource change data.
-    EXECUTE dbo.CaptureResourceIdsForChanges @Resources
+    EXECUTE dbo.CaptureResourceIdsForChanges @Resources, @ResourcesLake
 
   IF @TransactionId IS NOT NULL
     EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId