If output prefix is not specified but enable fhir store is true, only…

… upload data and don't write out to disk. PiperOrigin-RevId: 436854226
google · Mar 23, 2022 · 8eddcf2 · 8eddcf2
1 parent 2469013
commit 8eddcf2
Show file tree

Hide file tree

Showing 2 changed files with 97 additions and 22 deletions.
diff --git a/cmd/bcda_fetch/bcda_fetch.go b/cmd/bcda_fetch/bcda_fetch.go
@@ -37,7 +37,7 @@ import (
 var (
 	clientID                  = flag.String("client_id", "", "BCDA API client ID (required)")
 	clientSecret              = flag.String("client_secret", "", "BCDA API client secret (required)")
-	outputPrefix              = flag.String("output_prefix", "claims/claims_data", "Data output prefix")
+	outputPrefix              = flag.String("output_prefix", "", "Data output prefix. If unset, no file output will be written.")
 	useV2                     = flag.Bool("use_v2", false, "This indicates if the BCDA V2 API should be used, which returns R4 mapped data.")
 	rectify                   = flag.Bool("rectify", false, "This indicates that this program should attempt to rectify BCDA FHIR so that it is valid R4 FHIR.")
 	enableFHIRStore           = flag.Bool("enable_fhir_store", false, "If true, this enables write to GCP FHIR store. If true, all other fhir_store_* flags must be set.")
@@ -100,6 +100,10 @@ func mainWrapper(cfg mainWrapperConfig) error {
 		return errors.New("if enable_fhir_store is true, all other FHIR store related flags must be set")
 	}
 
+	if *outputPrefix == "" && !*enableFHIRStore {
+		log.Warningln("outputPrefix is not set and neither is enableFHIRStore: BCDA fetch will not produce any output.")
+	}
+
 	apiVersion := bcda.V1
 	if *useV2 {
 		apiVersion = bcda.V2
@@ -166,7 +170,11 @@ func mainWrapper(cfg mainWrapperConfig) error {
 
 	for r, urls := range jobStatus.ResultURLs {
 		for i, url := range urls {
-			filePrefix := fmt.Sprintf("%s_%s_%d", *outputPrefix, r, i)
+			filePrefix := ""
+			if *outputPrefix != "" {
+				filePrefix = fmt.Sprintf("%s_%s_%d", *outputPrefix, r, i)
+			}
+
 			r, err := getDataOrExit(cl, url, *clientID, *clientSecret)
 			if err != nil {
 				return err
@@ -236,9 +244,13 @@ func writeData(r io.Reader, filePrefix string) {
 }
 
 func rectifyAndWrite(r io.Reader, filePrefix string, cfg mainWrapperConfig) {
-		w, err := os.OpenFile(fmt.Sprintf("%s.ndjson", filePrefix), os.O_RDWR|os.O_CREATE, 0755)
-	if err != nil {
-		log.Exitf("Unable to create output file: %v", err)
+	var w io.Writer = nil
+	if filePrefix != "" {
+		var err error
+		w, err = os.OpenFile(fmt.Sprintf("%s.ndjson", filePrefix), os.O_RDWR|os.O_CREATE, 0755)
+		if err != nil {
+			log.Exitf("Unable to create output file: %v", err)
+		}
 	}
 
 	uploader := newFHIRStoreUploader(cfg.fhirStoreEndpoint)
@@ -259,8 +271,10 @@ func rectifyAndWrite(r io.Reader, filePrefix string, cfg mainWrapperConfig) {
 			uploader.Upload(fhirOut)
 		}
 
-		if _, err := w.Write(fhirOut); err != nil {
-			log.Exitf("issue during file write: %v", err)
+		if w != nil {
+			if _, err := w.Write(fhirOut); err != nil {
+				log.Exitf("issue during file write: %v", err)
+			}
 		}
 	}
 	// Since there is only one sender goroutine (this one), it should be safe to

diff --git a/cmd/bcda_fetch/bcda_fetch_test.go b/cmd/bcda_fetch/bcda_fetch_test.go
@@ -52,7 +52,9 @@ func TestMainWrapper(t *testing.T) {
 		fhirStoreFailures    bool
 		noFailOnUploadErrors bool
 		bcdaJobID            string
-		wantError            error
+		// unsetOutputPrefix sets the outputPrefix to empty string if true.
+		unsetOutputPrefix bool
+		wantError         error
 	}{
 		{
 			name:            "RectifyEnabledWithoutFHIRStoreBCDAV1",
@@ -253,6 +255,61 @@ func TestMainWrapper(t *testing.T) {
 			sinceFileExpectedContent: []byte("2020-12-09T11:00:00.123+00:00\n"),
 			bcdaJobID:                "999",
 		},
+		// Only testing cases with FHIR Store enabled for setting outputPrefix to ""
+		{
+			name:              "EmptyOutputPrefixWithRectifyEnabledWithFHIRStoreBCDAV1",
+			rectify:           true,
+			enableFHIRStore:   true,
+			apiVersion:        bcda.V1,
+			unsetOutputPrefix: true,
+		},
+		{
+			name:              "EmptyOutputPrefixWithRectifyEnabledWithFHIRStoreBCDAV2",
+			rectify:           true,
+			enableFHIRStore:   true,
+			apiVersion:        bcda.V2,
+			unsetOutputPrefix: true,
+		},
+		{
+			name:              "EmptyOutputPrefixWithSinceProvidedWithRectifyWithFHIRStoreBCDAV2WithBCDAJobId",
+			rectify:           true,
+			enableFHIRStore:   true,
+			apiVersion:        bcda.V2,
+			since:             "2006-01-02T15:04:05.000-07:00",
+			bcdaJobID:         "999",
+			unsetOutputPrefix: true,
+		},
+		{
+			name:              "EmptyOutputPrefixWithSinceProvidedWithRectifyWithFHIRStoreBCDAV1WithBCDAJobId",
+			rectify:           true,
+			enableFHIRStore:   true,
+			apiVersion:        bcda.V1,
+			since:             "2006-01-02T15:04:05.000-07:00",
+			bcdaJobID:         "999",
+			unsetOutputPrefix: true,
+		},
+		{
+			name:                     "EmptyOutputPrefixWithSinceFileProvidedWithBCDAV2WithBCDAJobIdWithFHIRStore",
+			rectify:                  true,
+			enableFHIRStore:          true,
+			apiVersion:               bcda.V2,
+			sinceFileContent:         []byte("2013-12-09T11:00:00.123+00:00\n2015-12-09T11:00:00.123+00:00\n"),
+			sinceFileLatestTimestamp: "2015-12-09T11:00:00.123+00:00",
+			sinceFileExpectedContent: []byte("2013-12-09T11:00:00.123+00:00\n2015-12-09T11:00:00.123+00:00\n2020-12-09T11:00:00.123+00:00\n"),
+			bcdaJobID:                "999",
+			unsetOutputPrefix:        true,
+		},
+		{
+			name:                     "EmptyOutputPrefixWithSinceFileProvidedWithBCDAV1WithBCDAJobIdWithFHIRStore",
+			rectify:                  true,
+			enableFHIRStore:          true,
+			apiVersion:               bcda.V1,
+			sinceFileContent:         []byte("2013-12-09T11:00:00.123+00:00\n2015-12-09T11:00:00.123+00:00\n"),
+			sinceFileLatestTimestamp: "2015-12-09T11:00:00.123+00:00",
+			sinceFileExpectedContent: []byte("2013-12-09T11:00:00.123+00:00\n2015-12-09T11:00:00.123+00:00\n2020-12-09T11:00:00.123+00:00\n"),
+			bcdaJobID:                "999",
+			unsetOutputPrefix:        true,
+		},
 		// TODO(b/226375559): see if we can generate some of the test cases above
 		// instead of having to spell them out explicitly.
 		// TODO(b/213365276): test that bcda V1 with rectify = true results in an
@@ -363,7 +420,9 @@ func TestMainWrapper(t *testing.T) {
 			defer SaveFlags().Restore()
 			flag.Set("client_id", "id")
 			flag.Set("client_secret", "secret")
-			flag.Set("output_prefix", outputPrefix)
+			if !tc.unsetOutputPrefix {
+				flag.Set("output_prefix", outputPrefix)
+			}
 			flag.Set("bcda_server_url", bcdaServer.URL)
 
 			flag.Set("fhir_store_gcp_project", gcpProject)
@@ -466,19 +525,21 @@ func TestMainWrapper(t *testing.T) {
 				}
 			}
 
-			for fileSuffix, wantData := range expectedFileSuffixToData {
-				fullPath := outputPrefix + fileSuffix
-				r, err := os.Open(fullPath)
-				if err != nil {
-					t.Errorf("unable to open file %s: %s", fullPath, err)
-				}
-				defer r.Close()
-				gotData, err := io.ReadAll(r)
-				if err != nil {
-					t.Errorf("error reading file %s: %v", fullPath, err)
-				}
-				if !cmp.Equal(normalizeJSON(t, gotData), normalizeJSON(t, wantData)) {
-					t.Errorf("mainWrapper unexpected ndjson output for file %s. got: %s, want: %s", fullPath, gotData, wantData)
+			if !tc.unsetOutputPrefix {
+				for fileSuffix, wantData := range expectedFileSuffixToData {
+					fullPath := outputPrefix + fileSuffix
+					r, err := os.Open(fullPath)
+					if err != nil {
+						t.Errorf("unable to open file %s: %s", fullPath, err)
+					}
+					defer r.Close()
+					gotData, err := io.ReadAll(r)
+					if err != nil {
+						t.Errorf("error reading file %s: %v", fullPath, err)
+					}
+					if !cmp.Equal(normalizeJSON(t, gotData), normalizeJSON(t, wantData)) {
+						t.Errorf("mainWrapper unexpected ndjson output for file %s. got: %s, want: %s", fullPath, gotData, wantData)
+					}
 				}
 			}