diff --git a/.gitignore b/.gitignore index aded8ded..4df61cf4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ output.sql MRCONSO.RRF *.zip coverage.xml -*.parquet valueset_data/ # Byte-compiled / optimized / DLL files diff --git a/cumulus_library/actions/uploader.py b/cumulus_library/actions/uploader.py index ad684e66..b7aef83f 100644 --- a/cumulus_library/actions/uploader.py +++ b/cumulus_library/actions/uploader.py @@ -14,7 +14,7 @@ def upload_data( progress: Progress, file_upload_progress: TaskID, file_path: Path, - version: int, + version: str, args: dict, ): """Fetches presigned url and uploads file to aggregator""" @@ -71,7 +71,10 @@ def upload_files(args: dict): if args["target"]: filtered_paths = [] for path in file_paths: - if any(study in str(path) for study in args["target"]): + if any( + path.parent.name == study and path.name.startswith(f"{study}__") + for study in args["target"] + ): filtered_paths.append(path) file_paths = filtered_paths diff --git a/tests/test_actions.py b/tests/test_actions.py index bb8ff43a..166d0082 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -1,4 +1,5 @@ import builtins +import contextlib import datetime import pathlib import shutil @@ -11,6 +12,7 @@ import requests import responses from freezegun import freeze_time +from responses import matchers from cumulus_library import base_utils, enums, errors, log_utils, study_manifest from cumulus_library.actions import ( @@ -410,6 +412,52 @@ def test_import_study(tmp_path, mock_db_config): ) +def do_upload( + *, + login_error: bool = False, + user: str = "user", + id_token: str = "id", + preview: bool = True, + raises: contextlib.AbstractContextManager = does_not_raise(), + status: int = 204, + version: str = "12345.0", + data_path: pathlib.Path | None = pathlib.Path.cwd() / "tests/test_data/upload/", +): + with raises: + if login_error: + responses.add(responses.POST, "https://upload.url.test/upload/", status=401) + else: + responses.add( + responses.POST, + "https://upload.url.test/upload/", + match=[ + matchers.json_params_matcher( + { + "study": "upload", + "data_package": "upload__count_synthea_patient", + "data_package_version": version, + "filename": f"{user}_upload__count_synthea_patient.parquet", + } + ) + ], + json={"url": "https://presigned.url.test", "fields": {"a": "b"}}, + ) + args = { + "data_path": data_path, + "id": id_token, + "preview": preview, + "target": ["upload"], + "url": "https://upload.url.test/upload/", + "user": user, + } + responses.add(responses.POST, "https://presigned.url.test", status=status) + uploader.upload_files(args) + if preview: + responses.assert_call_count("https://upload.url.test/upload/", 1) + elif raises == does_not_raise(): + responses.assert_call_count("https://upload.url.test/upload/", 2) + + @pytest.mark.parametrize( "user,id_token,status,login_error,preview,raises", [ @@ -451,29 +499,28 @@ def test_import_study(tmp_path, mock_db_config): ) @responses.activate def test_upload_data(user, id_token, status, preview, login_error, raises): - with raises: - if login_error: - responses.add(responses.POST, "https://upload.url.test/upload/", status=401) - else: - responses.add( - responses.POST, - "https://upload.url.test/upload/", - json={"url": "https://presigned.url.test", "fields": {"a": "b"}}, - ) - args = { - "data_path": pathlib.Path.cwd() / "tests/test_data/upload/", - "id": id_token, - "preview": preview, - "target": "core", - "url": "https://upload.url.test/upload/", - "user": user, - } - responses.add(responses.POST, "https://presigned.url.test", status=status) - uploader.upload_files(args) - if preview: - responses.assert_call_count("https://upload.url.test/upload/", 1) - elif raises == does_not_raise(): - responses.assert_call_count("https://upload.url.test/upload/", 2) + do_upload( + user=user, + id_token=id_token, + status=status, + preview=preview, + login_error=login_error, + raises=raises, + ) + + +def test_upload_data_no_path(): + with pytest.raises(SystemExit): + do_upload(data_path=None) + + +@responses.activate +def test_upload_data_no_version(tmp_path): + src = pathlib.Path.cwd() / "tests/test_data/upload/upload__count_synthea_patient.parquet" + dest = pathlib.Path(tmp_path) / "upload" + dest.mkdir() + shutil.copy(src, dest) + do_upload(data_path=dest, version="0") @pytest.mark.parametrize( diff --git a/tests/test_cli.py b/tests/test_cli.py index b9b0763c..79deeca0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -601,8 +601,12 @@ def test_cli_upload_studies(mock_glob, args, status, login_error, raises): def test_cli_upload_filter(mock_upload_data, mock_glob, args, calls): mock_glob.side_effect = [ [ - Path(str(Path(__file__).parent) + "/test_data/count_synthea_patient.parquet"), - Path(str(Path(__file__).parent) + "/other_data/count_synthea_patient.parquet"), + Path( + str(Path(__file__).parent) + "/test_data/test_data__count_synthea_patient.parquet" + ), + Path( + str(Path(__file__).parent) + "/other_data/other_data__count_synthea_patient.parquet" + ), ], ] cli.main(cli_args=args) diff --git a/tests/test_data/upload/count_synthea_patient.csv b/tests/test_data/upload/count_synthea_patient.csv deleted file mode 100644 index ce59d3f4..00000000 --- a/tests/test_data/upload/count_synthea_patient.csv +++ /dev/null @@ -1,254 +0,0 @@ -cnt,gender,age,race_display -1103,,, -1008,,,Not Hispanic or Latino -980,,,White -552,male,, -551,female,, -511,male,,Not Hispanic or Latino -500,female,,White -497,female,,Not Hispanic or Latino -480,male,,White -95,,,Hispanic or Latino -65,,,Black or African American -54,female,,Hispanic or Latino -43,male,,Black or African American -41,male,,Hispanic or Latino -22,,16, -22,female,,Black or African American -21,,74,Not Hispanic or Latino -21,,11, -21,,46, -21,,74,White -21,,74, -21,,46,Not Hispanic or Latino -20,,16,Not Hispanic or Latino -20,,2, -20,,,Asian -20,,22, -19,,48, -19,,2,Not Hispanic or Latino -19,,21, -19,,40, -19,,40,Not Hispanic or Latino -19,,20, -19,,46,White -19,,20,Not Hispanic or Latino -19,,14, -18,,16,White -18,,10,White -18,,48,Not Hispanic or Latino -18,,32, -18,,62, -18,,11,White -18,,62,Not Hispanic or Latino -18,,10, -18,,22,Not Hispanic or Latino -18,,11,Not Hispanic or Latino -17,,62,White -17,,63, -17,,14,Not Hispanic or Latino -17,,10,Not Hispanic or Latino -17,,53,White -17,,53, -17,,14,White -17,,32,White -17,,65,Not Hispanic or Latino -17,,20,White -17,,65, -16,,88,White -16,male,46,Not Hispanic or Latino -16,,22,White -16,,24,Not Hispanic or Latino -16,,40,White -16,,75, -16,,53,Not Hispanic or Latino -16,,4, -16,,88, -16,,24,White -16,,24, -16,male,46, -16,,48,White -16,,32,Not Hispanic or Latino -16,,33, -16,,61,Not Hispanic or Latino -16,,88,Not Hispanic or Latino -16,,61, -15,,3, -15,,25, -15,,43, -15,,4,Not Hispanic or Latino -15,,,Unknown -15,,33,White -15,male,46,White -15,,51, -15,,25,White -15,,65,White -15,,2,White -15,,58, -15,,21,White -14,,3,White -14,,63,White -14,,25,Not Hispanic or Latino -14,,5, -14,,,Native Hawaiian or Other Pacific Islander -14,,4,White -14,,5,White -14,,33,Not Hispanic or Latino -14,,51,White -14,female,75,White -14,,37, -14,,58,Not Hispanic or Latino -14,,27, -14,,75,White -14,,27,White -14,female,75, -14,,21,Not Hispanic or Latino -13,female,16,Not Hispanic or Latino -13,,18, -13,,50, -13,,75,Not Hispanic or Latino -13,,1, -13,,51,Not Hispanic or Latino -13,,82,White -13,,59, -13,,8, -13,female,75,Not Hispanic or Latino -13,,41, -13,,58,White -13,,18,Not Hispanic or Latino -13,,30, -13,,82, -13,,82,Not Hispanic or Latino -13,,15, -13,,3,Not Hispanic or Latino -13,,67, -13,female,16, -13,,27,Not Hispanic or Latino -13,,63,Not Hispanic or Latino -12,,66, -12,,43,Not Hispanic or Latino -12,,52,White -12,,8,White -12,,31,Not Hispanic or Latino -12,,30,White -12,,61,White -12,male,82,Not Hispanic or Latino -12,,44, -12,,31, -12,female,,Asian -12,male,82,White -12,,38, -12,,19,Not Hispanic or Latino -12,,9,Not Hispanic or Latino -12,,50,White -12,,64,Not Hispanic or Latino -12,,67,Not Hispanic or Latino -12,,37,White -12,,19, -12,,52, -12,,42,White -12,,9, -12,,64,White -12,male,2,Not Hispanic or Latino -12,,50,Not Hispanic or Latino -12,,42, -12,,64, -12,,8,Not Hispanic or Latino -12,male,11, -12,,59,Not Hispanic or Latino -12,,43,White -12,,6, -12,,6,Not Hispanic or Latino -12,,52,Not Hispanic or Latino -12,,5,Not Hispanic or Latino -12,,36, -12,male,82, -12,,1,White -12,,12, -12,,41,White -12,male,2, -11,,45, -11,,15,White -11,female,88,White -11,female,20, -11,,78, -11,,36,White -11,male,74,Not Hispanic or Latino -11,female,16,White -11,,35, -11,,45,Not Hispanic or Latino -11,female,40,Not Hispanic or Latino -11,,29, -11,,45,White -11,,68, -11,,67,White -11,male,48,Not Hispanic or Latino -11,female,63, -11,male,22, -11,,41,Not Hispanic or Latino -11,,7,Not Hispanic or Latino -11,,66,White -11,,37,Not Hispanic or Latino -11,female,88, -11,,55, -11,male,74,White -11,,30,Not Hispanic or Latino -11,,44,Not Hispanic or Latino -11,,6,White -11,,47,White -11,male,74, -11,female,88,Not Hispanic or Latino -11,,29,Not Hispanic or Latino -11,female,20,Not Hispanic or Latino -11,,12,Not Hispanic or Latino -11,,18,White -11,female,40, -11,,13, -11,,47, -11,male,11,Not Hispanic or Latino -11,male,22,Not Hispanic or Latino -11,,12,White -11,male,48, -11,,38,White -11,,7, -11,,77, -11,,15,Not Hispanic or Latino -11,,78,White -10,,69,Not Hispanic or Latino -10,,29,White -10,,26, -10,,26,White -10,,78,Not Hispanic or Latino -10,female,51, -10,female,33, -10,,39, -10,female,20,White -10,male,,Unknown -10,male,14, -10,,68,Not Hispanic or Latino -10,male,10, -10,female,74,Not Hispanic or Latino -10,male,10,Not Hispanic or Latino -10,female,74,White -10,male,65, -10,female,40,White -10,,57, -10,,69, -10,,17, -10,,17,Not Hispanic or Latino -10,,36,Not Hispanic or Latino -10,,66,Not Hispanic or Latino -10,,56, -10,,42,Not Hispanic or Latino -10,male,11,White -10,,35,White -10,,1,Not Hispanic or Latino -10,,55,Not Hispanic or Latino -10,female,21, -10,,57,White -10,male,65,Not Hispanic or Latino -10,female,74, -10,,7,White -10,,69,White -10,male,10,White -10,,68,White diff --git a/tests/test_data/upload/count_synthea_patient.parquet b/tests/test_data/upload/upload__count_synthea_patient.parquet similarity index 100% rename from tests/test_data/upload/count_synthea_patient.parquet rename to tests/test_data/upload/upload__count_synthea_patient.parquet diff --git a/tests/test_data/upload/upload__meta_version.parquet b/tests/test_data/upload/upload__meta_version.parquet new file mode 100644 index 00000000..76250385 Binary files /dev/null and b/tests/test_data/upload/upload__meta_version.parquet differ