Engagement/jessica/15406 cli download report

prime-router/docs/api/reports.yml

@@ -102,7 +102,36 @@ paths:
                 $ref: '#/components/schemas/Report'
         '500':
           description: Internal Server Error
-
+  /reports/download:
+    get:
+      summary: Downloads a message based on the report id
+      security:
+        - OAuth2: [ system_admin ]
+      parameters:
+        - in: query
+          name: reportId
+          description: The report id to look for to download.
+          schema:
+            type: string
+          required: true
+          example: e491f4fb-f2c5-4473-8db2-206ea04991e8
+        - in: query
+          name: removePII
+          description: Boolean that determines if PII will be removed from the message. If missing will default to true.
+            Required to be true if prod env.
+          required: false
+          schema:
+            type: boolean
+          example: true
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Report'
+        '500':
+          description: Internal Server Error
 # Building
 components:
   schemas:

prime-router/src/main/kotlin/azure/ReportFunction.kt

@@ -15,6 +15,7 @@ import gov.cdc.prime.router.ActionLogLevel
 import gov.cdc.prime.router.InvalidParamMessage
 import gov.cdc.prime.router.InvalidReportMessage
 import gov.cdc.prime.router.Options
+import gov.cdc.prime.router.ReportId
 import gov.cdc.prime.router.Sender
 import gov.cdc.prime.router.Sender.ProcessingType
 import gov.cdc.prime.router.SubmissionReceiver
@@ -24,13 +25,18 @@ import gov.cdc.prime.router.azure.observability.event.IReportStreamEventService
 import gov.cdc.prime.router.azure.observability.event.ReportStreamEventName
 import gov.cdc.prime.router.azure.observability.event.ReportStreamEventProperties
 import gov.cdc.prime.router.azure.observability.event.ReportStreamEventService
+import gov.cdc.prime.router.cli.PIIRemovalCommands
 import gov.cdc.prime.router.common.AzureHttpUtils.getSenderIP
+import gov.cdc.prime.router.common.Environment
 import gov.cdc.prime.router.common.JacksonMapperUtilities
+import gov.cdc.prime.router.fhirengine.utils.FhirTranscoder
 import gov.cdc.prime.router.history.azure.SubmissionsFacade
 import gov.cdc.prime.router.tokens.AuthenticatedClaims
+import gov.cdc.prime.router.tokens.Scope
 import gov.cdc.prime.router.tokens.authenticationFailure
 import gov.cdc.prime.router.tokens.authorizationFailure
 import org.apache.logging.log4j.kotlin.Logging
+import java.util.UUID
 
 private const val PROCESSING_TYPE_PARAMETER = "processing"
 
@@ -85,6 +91,73 @@ class ReportFunction(
         }
     }
 
+    /**
+     * GET report to download
+     *
+     * @see ../../../docs/api/reports.yml
+     */
+    @FunctionName("getMessagesFromTestBank")
+    fun downloadReport(
+        @HttpTrigger(
+            name = "downloadReport",
+            methods = [HttpMethod.GET],
+            authLevel = AuthorizationLevel.ANONYMOUS,
+            route = "reports/download"
+        ) request: HttpRequestMessage<String?>,
+    ): HttpResponseMessage {
+        val claims = AuthenticatedClaims.authenticate(request)
+        if (claims != null && claims.authorized(setOf(Scope.primeAdminScope))) {
+            val reportId = request.queryParameters[REPORT_ID_PARAMETER]
+            val removePIIRaw = request.queryParameters[REMOVE_PII]
+            var removePII = false
+            if (removePIIRaw.isNullOrBlank() || removePIIRaw.toBoolean()) {
+                removePII = true
+            }
+            if (reportId.isNullOrBlank()) {
+                return HttpUtilities.badRequestResponse(request, "Must provide a reportId.")
+            }
+            return processDownloadReport(
+                request,
+                ReportId.fromString(reportId),
+                removePII,
+                Environment.get().envName
+            )
+        }
+        return HttpUtilities.unauthorizedResponse(request)
+    }
+
+    fun processDownloadReport(
+        request: HttpRequestMessage<String?>,
+        reportId: UUID,
+        removePII: Boolean?,
+        envName: String,
+        databaseAccess: DatabaseAccess = DatabaseAccess(),
+    ): HttpResponseMessage {
+        val requestedReport = databaseAccess.fetchReportFile(reportId)
+
+        return if (requestedReport.bodyUrl != null && requestedReport.bodyUrl.toString().lowercase().endsWith("fhir")) {
+            val contents = BlobAccess.downloadBlobAsByteArray(requestedReport.bodyUrl)
+
+            val content = if (removePII == null || removePII) {
+                PIIRemovalCommands().removePii(FhirTranscoder.decode(contents.toString(Charsets.UTF_8)))
+            } else {
+                if (envName == "prod") {
+                    return HttpUtilities.badRequestResponse(request, "Must remove PII for messages from prod.")
+                }
+
+                val jsonObject = JacksonMapperUtilities.defaultMapper
+                    .readValue(contents.toString(Charsets.UTF_8), Any::class.java)
+                JacksonMapperUtilities.defaultMapper.writeValueAsString(jsonObject)
+            }
+
+            HttpUtilities.okJSONResponse(request, content)
+        } else if (requestedReport.bodyUrl == null) {
+            HttpUtilities.badRequestResponse(request, "The requested report does not exist.")
+        } else {
+            HttpUtilities.badRequestResponse(request, "The requested report is not fhir.")
+        }
+    }
+
     /**
      * The Waters API, in memory of Dr. Michael Waters
      * (The older version of this API is "/api/reports")

prime-router/src/main/kotlin/azure/RequestFunction.kt

@@ -23,6 +23,8 @@ const val ALLOW_DUPLICATES_PARAMETER = "allowDuplicate"
 const val TOPIC_PARAMETER = "topic"
 const val SCHEMA_PARAMETER = "schema"
 const val FORMAT_PARAMETER = "format"
+const val REPORT_ID_PARAMETER = "reportId"
+const val REMOVE_PII = "removePII"
 
 /**
  * Base class for ReportFunction and ValidateFunction

prime-router/src/main/kotlin/cli/PIIRemovalCommands.kt

@@ -56,8 +56,16 @@ class PIIRemovalCommands : CliktCommand(
         if (inputFile.extension.uppercase() != "FHIR") {
             throw CliktError("File ${inputFile.absolutePath} is not a FHIR file.")
         }
-        var bundle = FhirTranscoder.decode(contents)
+        val bundle = FhirTranscoder.decode(contents)
 
+        // Write the output to the screen or a file.
+        if (outputFile != null) {
+            outputFile!!.writeText(removePii(bundle), Charsets.UTF_8)
+        }
+        echo("Wrote output to ${outputFile!!.absolutePath}")
+    }
+
+    fun removePii(bundle: Bundle): String {
         bundle.entry.map { it.resource }.filterIsInstance<Patient>()
             .forEach { patient ->
                 patient.name.forEach { name ->
@@ -76,16 +84,16 @@ class PIIRemovalCommands : CliktCommand(
 
         bundle.entry.map { it.resource }.filterIsInstance<Organization>()
             .forEach { organization ->
-            organization.address.forEach { address ->
-                address.line = mutableListOf(StringType(getFakeValueForElementCall("STREET")))
-            }
-            organization.telecom.forEach { telecom ->
-                handleTelecom(telecom)
-            }
-            organization.contact.forEach { contact ->
-               handleOrganizationalContact(contact)
+                organization.address.forEach { address ->
+                    address.line = mutableListOf(StringType(getFakeValueForElementCall("STREET")))
+                }
+                organization.telecom.forEach { telecom ->
+                    handleTelecom(telecom)
+                }
+                organization.contact.forEach { contact ->
+                    handleOrganizationalContact(contact)
+                }
             }
-        }
 
         bundle.entry.map { it.resource }.filterIsInstance<Practitioner>()
             .forEach { practitioner ->
@@ -103,18 +111,14 @@ class PIIRemovalCommands : CliktCommand(
                 }
             }
 
-        bundle = FhirTransformer("classpath:/metadata/fhir_transforms/common/remove-pii-enrichment.yml").process(bundle)
+        val bundleAfterTransform = FhirTransformer(
+            "classpath:/metadata/fhir_transforms/common/remove-pii-enrichment.yml"
+        ).process(bundle)
 
         val jsonObject = JacksonMapperUtilities.defaultMapper
-            .readValue(FhirTranscoder.encode(bundle), Any::class.java)
-        var prettyText = JacksonMapperUtilities.defaultMapper.writeValueAsString(jsonObject)
-        prettyText = replaceIds(bundle, prettyText)
-
-        // Write the output to the screen or a file.
-        if (outputFile != null) {
-            outputFile!!.writeText(prettyText, Charsets.UTF_8)
-        }
-        echo("Wrote output to ${outputFile!!.absolutePath}")
+            .readValue(FhirTranscoder.encode(bundleAfterTransform), Any::class.java)
+        val prettyText = JacksonMapperUtilities.defaultMapper.writeValueAsString(jsonObject)
+        return replaceIds(bundleAfterTransform, prettyText)
     }
 
     /**
@@ -185,8 +189,10 @@ class PIIRemovalCommands : CliktCommand(
             bundle,
             path
         ).forEach { resourceId ->
-            val newIdentifier = getFakeValueForElementCall("UUID")
-            return prettyText.replace(resourceId.primitiveValue(), newIdentifier, true)
+            if (resourceId.primitiveValue() != null) {
+                val newIdentifier = getFakeValueForElementCall("UUID")
+                return prettyText.replace(resourceId.primitiveValue(), newIdentifier, true)
+            }
         }
         return prettyText
     }

prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-address.yml

@@ -2,8 +2,9 @@ elements:
   # removing the street address is more complicated because it is a list so we will do this in code
 
   - name: pii-removal-street-address2
+    condition: "%resource.extension(`https://reportstream.cdc.gov/fhir/StructureDefinition/xad-address`).exits()"
     value: [ 'getFakeValueForElement("STREET_ADDRESS_2")' ]
-    bundleProperty: '%resource.extension(%`rsext-xad-address`).extension.where(url = "XAD.2").value'
+    bundleProperty: '%resource.extension(`https://reportstream.cdc.gov/fhir/StructureDefinition/xad-address`).extension.where(url = "XAD.2").value'
 
   - name: pii-removal-city
     value: [ 'getFakeValueForElement("CITY",%resource.state)' ]

prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-name.yml

@@ -6,5 +6,6 @@ elements:
   #  removing a given name is more complicated because it is a list so we will do this in code
 
   - name: pii-removal-middle-name
+    condition: '%resource.extension("https://reportstream.cdc.gov/fhir/StructureDefinition/xpn-human-name").exists()'
     value: [ 'getFakeValueForElement("PERSON_GIVEN_NAME")' ]
-    bundleProperty: '%resource.extension(%`rsext-xpn-human-name`).extension.where(url="XPN.3").value'
+    bundleProperty: '%resource.extension("https://reportstream.cdc.gov/fhir/StructureDefinition/xpn-human-name").extension("XPN.2").value[x]'

prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-telecom.yml

@@ -2,7 +2,7 @@ elements:
   - name: pii-removal-phone-area-code
     condition: "%resource.where(system = 'phone')"
     value: [ 'getFakeValueForElement("TELEPHONE").substring(0,3)' ]
-    bundleProperty: '%resource.extension(%`ext-contactpoint-area`).value'
+    bundleProperty: '%resource.extension(`https://reportstream.cdc.gov/fhir/StructureDefinition/contactpoint-area`).value[x]'
 
   - name: pii-removal-local-phone
     condition: "%resource.where(system = 'phone')"

prime-router/src/test/kotlin/azure/ReportFunctionTests.kt

@@ -24,6 +24,7 @@ import gov.cdc.prime.router.Topic
 import gov.cdc.prime.router.TopicReceiver
 import gov.cdc.prime.router.UniversalPipelineSender
 import gov.cdc.prime.router.azure.db.enums.TaskAction
+import gov.cdc.prime.router.azure.db.tables.pojos.ReportFile
 import gov.cdc.prime.router.history.DetailedSubmissionHistory
 import gov.cdc.prime.router.history.azure.SubmissionsFacade
 import gov.cdc.prime.router.serializers.Hl7Serializer
@@ -45,6 +46,7 @@ import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import java.time.OffsetDateTime
 import java.util.UUID
+import kotlin.test.assertFailsWith
 
 class ReportFunctionTests {
     val dataProvider = MockDataProvider { emptyArray<MockResult>() }
@@ -172,6 +174,11 @@ class ReportFunctionTests {
         "05D2222542&ISO||445297001^Swab of internal nose^SCT^^^^2.67||||53342003^Internal nose structure" +
         " (body structure)^SCT^^^^2020-09-01|||||||||202108020000-0500|20210802000006.0000-0500"
 
+    @Suppress("ktlint:standard:max-line-length")
+    val fhirReport = """{"resourceType":"Bundle","id":"1667861767830636000.7db38d22-b713-49fc-abfa-2edba9c12347",
+        |"meta":{"lastUpdated":"2022-11-07T22:56:07.832+00:00"},"identifier":{"value":"1234d1d1-95fe-462c-8ac6-46728dba581c"},"type":"message","timestamp":"2021-08-03T13:15:11.015+00:00","entry":[{"fullUrl":"Observation/d683b42a-bf50-45e8-9fce-6c0531994f09","resource":{"resourceType":"Observation","id":"d683b42a-bf50-45e8-9fce-6c0531994f09","status":"final","code":{"coding":[{"system":"http://loinc.org","code":"80382-5"}],"text":"Flu A"},"subject":{"reference":"Patient/9473889b-b2b9-45ac-a8d8-191f27132912"},"performer":[{"reference":"Organization/1a0139b9-fc23-450b-9b6c-cd081e5cea9d"}],"valueCodeableConcept":{"coding":[{"system":"http://snomed.info/sct","code":"260373001","display":"Detected"}]},"interpretation":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0078","code":"A","display":"Abnormal"}]}],"method":{"extension":[{"url":"https://reportstream.cdc.gov/fhir/StructureDefinition/testkit-name-id","valueCoding":{"code":"BD Veritor System for Rapid Detection of SARS-CoV-2 & Flu A+B_Becton, Dickinson and Company (BD)"}},{"url":"https://reportstream.cdc.gov/fhir/StructureDefinition/equipment-uid","valueCoding":{"code":"BD Veritor System for Rapid Detection of SARS-CoV-2 & Flu A+B_Becton, Dickinson and Company (BD)"}}],"coding":[{"display":"BD Veritor System for Rapid Detection of SARS-CoV-2 & Flu A+B*"}]},"specimen":{"reference":"Specimen/52a582e4-d389-42d0-b738-bee51cf5244d"},"device":{"reference":"Device/78dc4d98-2958-43a3-a445-76ceef8c0698"}}}]}
+""".trimMargin()
+
     private fun makeEngine(metadata: Metadata, settings: SettingsProvider): WorkflowEngine = spyk(
         WorkflowEngine.Builder().metadata(metadata).settingsProvider(settings).databaseAccess(accessSpy)
             .blobAccess(blobMock).queueAccess(queueMock).hl7Serializer(serializer).build()
@@ -730,4 +737,109 @@ class ReportFunctionTests {
             reportFunc.extractPayloadName(mockHttpRequest)
         }
     }
+
+    @Test
+    fun `No report`() {
+        val mockDb = mockk<DatabaseAccess>()
+        val reportId = UUID.randomUUID()
+        every { mockDb.fetchReportFile(reportId, null, null) } throws (IllegalStateException())
+        assertFailsWith<IllegalStateException>(
+            block = {
+                ReportFunction().processDownloadReport(MockHttpRequestMessage(), reportId, true, "local", mockDb)
+            }
+        )
+    }
+
+    @Test
+    fun `Report found, PII removal`() {
+        val reportFile = ReportFile()
+        reportFile.bodyUrl = "fakeurl.fhir"
+        mockkObject(AuthenticatedClaims)
+        val mockDb = mockk<DatabaseAccess>()
+        mockkClass(BlobAccess::class)
+        mockkObject(BlobAccess.Companion)
+        every { BlobAccess.Companion.getBlobConnection(any()) } returns "testconnection"
+        val blobConnectionInfo = mockk<BlobAccess.BlobContainerMetadata>()
+        every { blobConnectionInfo.getBlobEndpoint() } returns "http://endpoint/metadata"
+        every { BlobAccess.downloadBlobAsByteArray(any<String>()) } returns fhirReport.toByteArray(Charsets.UTF_8)
+
+        val result = ReportFunction().processDownloadReport(
+            MockHttpRequestMessage(),
+            UUID.randomUUID(),
+            true,
+            "local",
+            mockDb
+        )
+
+        assert(result.body.toString().contains("1667861767830636000.7db38d22-b713-49fc-abfa-2edba9c12347"))
+    }
+
+    @Test
+    fun `Report found, asked for no removal on prod`() {
+        val reportFile = ReportFile()
+        reportFile.bodyUrl = "fakeurl.fhir"
+        mockkObject(AuthenticatedClaims)
+        val mockDb = mockk<DatabaseAccess>()
+        mockkClass(BlobAccess::class)
+        mockkObject(BlobAccess.Companion)
+        every { BlobAccess.Companion.getBlobConnection(any()) } returns "testconnection"
+        val blobConnectionInfo = mockk<BlobAccess.BlobContainerMetadata>()
+        every { blobConnectionInfo.getBlobEndpoint() } returns "http://endpoint/metadata"
+        every { BlobAccess.downloadBlobAsByteArray(any<String>()) } returns fhirReport.toByteArray(Charsets.UTF_8)
+        every { mockDb.fetchReportFile(reportId = any(), null, null) } returns reportFile
+
+        val result = ReportFunction().processDownloadReport(
+            MockHttpRequestMessage(),
+            UUID.randomUUID(),
+            false,
+            "prod",
+            mockDb
+        )
+
+        assert(result.status.equals(HttpStatus.BAD_REQUEST))
+    }
+
+    @Test
+    fun `valid access token, report found, no PII removal`() {
+        val reportFile = ReportFile()
+        reportFile.bodyUrl = "fakeurl.fhir"
+        mockkObject(AuthenticatedClaims)
+        val mockDb = mockk<DatabaseAccess>()
+        mockkClass(BlobAccess::class)
+        mockkObject(BlobAccess.Companion)
+        every { BlobAccess.Companion.getBlobConnection(any()) } returns "testconnection"
+        val blobConnectionInfo = mockk<BlobAccess.BlobContainerMetadata>()
+        every { blobConnectionInfo.getBlobEndpoint() } returns "http://endpoint/metadata"
+        every { BlobAccess.downloadBlobAsByteArray(any<String>()) } returns fhirReport.toByteArray(Charsets.UTF_8)
+        every { mockDb.fetchReportFile(reportId = any(), null, null) } returns reportFile
+
+        val result = ReportFunction().processDownloadReport(
+            MockHttpRequestMessage(),
+            UUID.randomUUID(),
+            false,
+            "local",
+            mockDb
+        )
+
+        assert(result.body.toString().contains("1667861767830636000.7db38d22-b713-49fc-abfa-2edba9c12347"))
+    }
+
+    @Test
+    fun `valid access token, report found, body URL not FHIR`() {
+        val reportFile = ReportFile()
+        reportFile.bodyUrl = "fakeurl.hl7"
+        mockkObject(AuthenticatedClaims)
+        val mockDb = mockk<DatabaseAccess>()
+        every { mockDb.fetchReportFile(reportId = any(), null, null) } returns reportFile
+
+        val result = ReportFunction().processDownloadReport(
+            MockHttpRequestMessage(),
+            UUID.randomUUID(),
+            false,
+            "local",
+            mockDb
+        )
+
+        assert(result.status.equals(HttpStatus.BAD_REQUEST))
+    }
 }