diff --git a/prime-router/docs/api/reports.yml b/prime-router/docs/api/reports.yml index e3475f53e21..2a860227aa8 100644 --- a/prime-router/docs/api/reports.yml +++ b/prime-router/docs/api/reports.yml @@ -118,6 +118,36 @@ paths: $ref: '#/components/schemas/Report' '500': description: Internal Server Error + /reports/download: + get: + summary: Downloads a message based on the report id + security: + - OAuth2: [ system_admin ] + parameters: + - in: query + name: reportId + description: The report id to look for to download. + schema: + type: string + required: true + example: e491f4fb-f2c5-4473-8db2-206ea04991e8 + - in: query + name: removePII + description: Boolean that determines if PII will be removed from the message. If missing will default to true. + Required to be true if prod env. + required: false + schema: + type: boolean + example: true + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/Report' + '500': + description: Internal Server Error # Building components: schemas: diff --git a/prime-router/src/main/kotlin/azure/ReportFunction.kt b/prime-router/src/main/kotlin/azure/ReportFunction.kt index 99168f9f3d9..11adb9fea4c 100644 --- a/prime-router/src/main/kotlin/azure/ReportFunction.kt +++ b/prime-router/src/main/kotlin/azure/ReportFunction.kt @@ -18,6 +18,7 @@ import gov.cdc.prime.router.ActionLogLevel import gov.cdc.prime.router.InvalidParamMessage import gov.cdc.prime.router.InvalidReportMessage import gov.cdc.prime.router.Options +import gov.cdc.prime.router.ReportId import gov.cdc.prime.router.Sender import gov.cdc.prime.router.Sender.ProcessingType import gov.cdc.prime.router.SubmissionReceiver @@ -29,14 +30,18 @@ import gov.cdc.prime.router.azure.observability.event.IReportStreamEventService import gov.cdc.prime.router.azure.observability.event.ReportStreamEventName import gov.cdc.prime.router.azure.observability.event.ReportStreamEventProperties import gov.cdc.prime.router.azure.observability.event.ReportStreamEventService +import gov.cdc.prime.router.cli.PIIRemovalCommands import gov.cdc.prime.router.common.AzureHttpUtils.getSenderIP +import gov.cdc.prime.router.common.Environment import gov.cdc.prime.router.common.JacksonMapperUtilities +import gov.cdc.prime.router.fhirengine.utils.FhirTranscoder import gov.cdc.prime.router.history.azure.SubmissionsFacade import gov.cdc.prime.router.tokens.AuthenticatedClaims import gov.cdc.prime.router.tokens.Scope import gov.cdc.prime.router.tokens.authenticationFailure import gov.cdc.prime.router.tokens.authorizationFailure import org.apache.logging.log4j.kotlin.Logging +import java.util.UUID private const val PROCESSING_TYPE_PARAMETER = "processing" @@ -155,6 +160,70 @@ class ReportFunction( var reportBody: String, ) + /** + * GET report to download + * + * @see ../../../docs/api/reports.yml + */ + @FunctionName("downloadReport") + fun downloadReport( + @HttpTrigger( + name = "downloadReport", + methods = [HttpMethod.GET], + authLevel = AuthorizationLevel.FUNCTION, + route = "reports/download" + ) request: HttpRequestMessage, + ): HttpResponseMessage { + val reportId = request.queryParameters[REPORT_ID_PARAMETER] + val removePIIRaw = request.queryParameters[REMOVE_PII] + var removePII = false + if (removePIIRaw.isNullOrBlank() || removePIIRaw.toBoolean()) { + removePII = true + } + if (reportId.isNullOrBlank()) { + return HttpUtilities.badRequestResponse(request, "Must provide a reportId.") + } + return processDownloadReport( + request, + ReportId.fromString(reportId), + removePII, + Environment.get().envName + ) + } + + fun processDownloadReport( + request: HttpRequestMessage, + reportId: UUID, + removePII: Boolean?, + envName: String, + databaseAccess: DatabaseAccess = DatabaseAccess(), + piiRemovalCommands: PIIRemovalCommands = PIIRemovalCommands(), + ): HttpResponseMessage { + val requestedReport = databaseAccess.fetchReportFile(reportId) + + return if (requestedReport.bodyUrl != null && requestedReport.bodyUrl.toString().lowercase().endsWith("fhir")) { + val contents = BlobAccess.downloadBlobAsByteArray(requestedReport.bodyUrl) + + val content = if (removePII == null || removePII) { + piiRemovalCommands.removePii(FhirTranscoder.decode(contents.toString(Charsets.UTF_8))) + } else { + if (envName == "prod") { + return HttpUtilities.badRequestResponse(request, "Must remove PII for messages from prod.") + } + + val jsonObject = JacksonMapperUtilities.defaultMapper + .readValue(contents.toString(Charsets.UTF_8), Any::class.java) + JacksonMapperUtilities.defaultMapper.writeValueAsString(jsonObject) + } + + HttpUtilities.okJSONResponse(request, content) + } else if (requestedReport.bodyUrl == null) { + HttpUtilities.badRequestResponse(request, "The requested report does not exist.") + } else { + HttpUtilities.badRequestResponse(request, "The requested report is not fhir.") + } + } + /** * The Waters API, in memory of Dr. Michael Waters * (The older version of this API is "/api/reports") diff --git a/prime-router/src/main/kotlin/azure/RequestFunction.kt b/prime-router/src/main/kotlin/azure/RequestFunction.kt index 47608913daf..e892befec42 100644 --- a/prime-router/src/main/kotlin/azure/RequestFunction.kt +++ b/prime-router/src/main/kotlin/azure/RequestFunction.kt @@ -23,6 +23,8 @@ const val ALLOW_DUPLICATES_PARAMETER = "allowDuplicate" const val TOPIC_PARAMETER = "topic" const val SCHEMA_PARAMETER = "schema" const val FORMAT_PARAMETER = "format" +const val REPORT_ID_PARAMETER = "reportId" +const val REMOVE_PII = "removePII" /** * Base class for ReportFunction and ValidateFunction diff --git a/prime-router/src/main/kotlin/cli/PIIRemovalCommands.kt b/prime-router/src/main/kotlin/cli/PIIRemovalCommands.kt index c74cb829cc1..09d450567dd 100644 --- a/prime-router/src/main/kotlin/cli/PIIRemovalCommands.kt +++ b/prime-router/src/main/kotlin/cli/PIIRemovalCommands.kt @@ -56,8 +56,16 @@ class PIIRemovalCommands : CliktCommand( if (inputFile.extension.uppercase() != "FHIR") { throw CliktError("File ${inputFile.absolutePath} is not a FHIR file.") } - var bundle = FhirTranscoder.decode(contents) + val bundle = FhirTranscoder.decode(contents) + // Write the output to the screen or a file. + if (outputFile != null) { + outputFile!!.writeText(removePii(bundle), Charsets.UTF_8) + } + echo("Wrote output to ${outputFile!!.absolutePath}") + } + + fun removePii(bundle: Bundle): String { bundle.entry.map { it.resource }.filterIsInstance() .forEach { patient -> patient.name.forEach { name -> @@ -76,16 +84,16 @@ class PIIRemovalCommands : CliktCommand( bundle.entry.map { it.resource }.filterIsInstance() .forEach { organization -> - organization.address.forEach { address -> - address.line = mutableListOf(StringType(getFakeValueForElementCall("STREET"))) - } - organization.telecom.forEach { telecom -> - handleTelecom(telecom) - } - organization.contact.forEach { contact -> - handleOrganizationalContact(contact) + organization.address.forEach { address -> + address.line = mutableListOf(StringType(getFakeValueForElementCall("STREET"))) + } + organization.telecom.forEach { telecom -> + handleTelecom(telecom) + } + organization.contact.forEach { contact -> + handleOrganizationalContact(contact) + } } - } bundle.entry.map { it.resource }.filterIsInstance() .forEach { practitioner -> @@ -103,18 +111,14 @@ class PIIRemovalCommands : CliktCommand( } } - bundle = FhirTransformer("classpath:/metadata/fhir_transforms/common/remove-pii-enrichment.yml").process(bundle) + val bundleAfterTransform = FhirTransformer( + "classpath:/metadata/fhir_transforms/common/remove-pii-enrichment.yml" + ).process(bundle) val jsonObject = JacksonMapperUtilities.defaultMapper - .readValue(FhirTranscoder.encode(bundle), Any::class.java) - var prettyText = JacksonMapperUtilities.defaultMapper.writeValueAsString(jsonObject) - prettyText = replaceIds(bundle, prettyText) - - // Write the output to the screen or a file. - if (outputFile != null) { - outputFile!!.writeText(prettyText, Charsets.UTF_8) - } - echo("Wrote output to ${outputFile!!.absolutePath}") + .readValue(FhirTranscoder.encode(bundleAfterTransform), Any::class.java) + val prettyText = JacksonMapperUtilities.defaultMapper.writeValueAsString(jsonObject) + return replaceIds(bundleAfterTransform, prettyText) } /** @@ -185,8 +189,10 @@ class PIIRemovalCommands : CliktCommand( bundle, path ).forEach { resourceId -> - val newIdentifier = getFakeValueForElementCall("UUID") - return prettyText.replace(resourceId.primitiveValue(), newIdentifier, true) + if (resourceId.primitiveValue() != null) { + val newIdentifier = getFakeValueForElementCall("UUID") + return prettyText.replace(resourceId.primitiveValue(), newIdentifier, true) + } } return prettyText } diff --git a/prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-address.yml b/prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-address.yml index 52117eab502..bb5c515cea8 100644 --- a/prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-address.yml +++ b/prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-address.yml @@ -2,8 +2,9 @@ elements: # removing the street address is more complicated because it is a list so we will do this in code - name: pii-removal-street-address2 + condition: '%resource.extension("https://reportstream.cdc.gov/fhir/StructureDefinition/xad-address").extension.where(url = "XAD.2")' value: [ 'getFakeValueForElement("STREET_ADDRESS_2")' ] - bundleProperty: '%resource.extension(%`rsext-xad-address`).extension.where(url = "XAD.2").value' + bundleProperty: '%resource.extension("https://reportstream.cdc.gov/fhir/StructureDefinition/xad-address").extension.where(url = "XAD.2").value' - name: pii-removal-city value: [ 'getFakeValueForElement("CITY",%resource.state)' ] diff --git a/prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-name.yml b/prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-name.yml index 4968f5eac5a..546e4581574 100644 --- a/prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-name.yml +++ b/prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-name.yml @@ -6,5 +6,6 @@ elements: # removing a given name is more complicated because it is a list so we will do this in code - name: pii-removal-middle-name + condition: '%resource.extension("https://reportstream.cdc.gov/fhir/StructureDefinition/xpn-human-name").exists()' value: [ 'getFakeValueForElement("PERSON_GIVEN_NAME")' ] - bundleProperty: '%resource.extension(%`rsext-xpn-human-name`).extension.where(url="XPN.3").value' \ No newline at end of file + bundleProperty: '%resource.extension("https://reportstream.cdc.gov/fhir/StructureDefinition/xpn-human-name").extension("XPN.2").value[x]' \ No newline at end of file diff --git a/prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-telecom.yml b/prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-telecom.yml index fef6b43d8a6..ce9dc465a8b 100644 --- a/prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-telecom.yml +++ b/prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-telecom.yml @@ -2,7 +2,7 @@ elements: - name: pii-removal-phone-area-code condition: "%resource.where(system = 'phone')" value: [ 'getFakeValueForElement("TELEPHONE").substring(0,3)' ] - bundleProperty: '%resource.extension(%`ext-contactpoint-area`).value' + bundleProperty: '%resource.extension(`https://reportstream.cdc.gov/fhir/StructureDefinition/contactpoint-area`).value[x]' - name: pii-removal-local-phone condition: "%resource.where(system = 'phone')" diff --git a/prime-router/src/test/kotlin/azure/ReportFunctionTests.kt b/prime-router/src/test/kotlin/azure/ReportFunctionTests.kt index 74ed50419a3..7581a1210ad 100644 --- a/prime-router/src/test/kotlin/azure/ReportFunctionTests.kt +++ b/prime-router/src/test/kotlin/azure/ReportFunctionTests.kt @@ -34,6 +34,8 @@ import gov.cdc.prime.router.TopicReceiver import gov.cdc.prime.router.UniversalPipelineSender import gov.cdc.prime.router.azure.BlobAccess.BlobContainerMetadata import gov.cdc.prime.router.azure.db.enums.TaskAction +import gov.cdc.prime.router.azure.db.tables.pojos.ReportFile +import gov.cdc.prime.router.cli.PIIRemovalCommands import gov.cdc.prime.router.history.DetailedSubmissionHistory import gov.cdc.prime.router.history.azure.SubmissionsFacade import gov.cdc.prime.router.serializers.Hl7Serializer @@ -57,6 +59,7 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import java.time.OffsetDateTime import java.util.UUID +import kotlin.test.assertFailsWith class ReportFunctionTests { val dataProvider = MockDataProvider { emptyArray() } @@ -842,4 +845,142 @@ class ReportFunctionTests { ) + "]" ) } + + @Test + fun `No report`() { + val mockDb = mockk() + val reportId = UUID.randomUUID() + every { mockDb.fetchReportFile(reportId, null, null) } throws (IllegalStateException()) + val metadata = UnitTestUtils.simpleMetadata + val settings = FileSettings().loadOrganizations(oneOrganization) + val actionHistory = spyk(ActionHistory(TaskAction.receive)) + assertFailsWith( + block = { + ReportFunction( + makeEngine(metadata, settings), + actionHistory + ).processDownloadReport( + MockHttpRequestMessage(), + reportId, + true, + "local", + mockDb + ) + } + ) + } + + @Test + fun `Report found, PII removal`() { + val reportFile = ReportFile() + reportFile.bodyUrl = "fakeurl.fhir" + mockkObject(AuthenticatedClaims) + val mockDb = mockk() + mockkClass(BlobAccess::class) + mockkObject(BlobAccess.Companion) + every { BlobAccess.Companion.getBlobConnection(any()) } returns "testconnection" + val blobConnectionInfo = mockk() + every { blobConnectionInfo.getBlobEndpoint() } returns "http://endpoint/metadata" + every { BlobAccess.downloadBlobAsByteArray(any()) } returns fhirReport.toByteArray(Charsets.UTF_8) + val reportId = UUID.randomUUID() + every { mockDb.fetchReportFile(reportId, null, null) } returns reportFile + val piiRemovalCommands = mockkClass(PIIRemovalCommands::class) + every { piiRemovalCommands.removePii(any()) } returns fhirReport + + val metadata = UnitTestUtils.simpleMetadata + val settings = FileSettings().loadOrganizations(oneOrganization) + val actionHistory = spyk(ActionHistory(TaskAction.receive)) + + val result = ReportFunction(makeEngine(metadata, settings), actionHistory).processDownloadReport( + MockHttpRequestMessage(), + reportId, + true, + "local", + mockDb, + piiRemovalCommands + ) + + assert(result.body.toString().contains("1667861767830636000.7db38d22-b713-49fc-abfa-2edba9c12347")) + } + + @Test + fun `Report found, asked for no removal on prod`() { + val reportFile = ReportFile() + reportFile.bodyUrl = "fakeurl.fhir" + mockkObject(AuthenticatedClaims) + val mockDb = mockk() + mockkClass(BlobAccess::class) + mockkObject(BlobAccess.Companion) + every { BlobAccess.Companion.getBlobConnection(any()) } returns "testconnection" + val blobConnectionInfo = mockk() + every { blobConnectionInfo.getBlobEndpoint() } returns "http://endpoint/metadata" + every { BlobAccess.downloadBlobAsByteArray(any()) } returns fhirReport.toByteArray(Charsets.UTF_8) + every { mockDb.fetchReportFile(reportId = any(), null, null) } returns reportFile + + val metadata = UnitTestUtils.simpleMetadata + val settings = FileSettings().loadOrganizations(oneOrganization) + val actionHistory = spyk(ActionHistory(TaskAction.receive)) + + val result = ReportFunction(makeEngine(metadata, settings), actionHistory).processDownloadReport( + MockHttpRequestMessage(), + UUID.randomUUID(), + false, + "prod", + mockDb + ) + + assert(result.status.equals(HttpStatus.BAD_REQUEST)) + } + + @Test + fun `valid access token, report found, no PII removal`() { + val reportFile = ReportFile() + reportFile.bodyUrl = "fakeurl.fhir" + mockkObject(AuthenticatedClaims) + val mockDb = mockk() + mockkClass(BlobAccess::class) + mockkObject(BlobAccess.Companion) + every { BlobAccess.Companion.getBlobConnection(any()) } returns "testconnection" + val blobConnectionInfo = mockk() + every { blobConnectionInfo.getBlobEndpoint() } returns "http://endpoint/metadata" + every { BlobAccess.downloadBlobAsByteArray(any()) } returns fhirReport.toByteArray(Charsets.UTF_8) + every { mockDb.fetchReportFile(reportId = any(), null, null) } returns reportFile + + val metadata = UnitTestUtils.simpleMetadata + val settings = FileSettings().loadOrganizations(oneOrganization) + val actionHistory = spyk(ActionHistory(TaskAction.receive)) + + val result = ReportFunction(makeEngine(metadata, settings), actionHistory).processDownloadReport( + MockHttpRequestMessage(), + UUID.randomUUID(), + false, + "local", + mockDb + ) + + assert(result.body.toString().contains("1667861767830636000.7db38d22-b713-49fc-abfa-2edba9c12347")) + } + + @Test + fun `valid access token, report found, body URL not FHIR`() { + val reportFile = ReportFile() + reportFile.bodyUrl = "fakeurl.hl7" + mockkObject(AuthenticatedClaims) + val mockDb = mockk() + every { mockDb.fetchReportFile(reportId = any(), null, null) } returns reportFile + + val metadata = UnitTestUtils.simpleMetadata + val settings = FileSettings().loadOrganizations(oneOrganization) + val actionHistory = spyk(ActionHistory(TaskAction.receive)) + + val result = ReportFunction(makeEngine(metadata, settings), actionHistory).processDownloadReport( + MockHttpRequestMessage(), + UUID.randomUUID(), + false, + "local", + mockDb + ) + + assert(result.status.equals(HttpStatus.BAD_REQUEST)) + } } \ No newline at end of file