From 3b9f92a2f7f1ebdddb74431b1b092c19fe432374 Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Fri, 20 Sep 2024 09:17:01 -0400 Subject: [PATCH 1/7] Update unit test data with changes to hipscat-import --- .github/workflows/build-documentation.yml | 2 +- .github/workflows/pre-commit-ci.yml | 2 +- .github/workflows/testing-and-coverage.yml | 2 +- tests/data/generate_data.ipynb | 133 ++++++++++-------- .../small_sky/Norder=0/Dir=0/Npix=11.parquet | Bin 8880 -> 8850 bytes tests/data/small_sky/_common_metadata | Bin 4018 -> 3994 bytes tests/data/small_sky/_metadata | Bin 5130 -> 5094 bytes tests/data/small_sky/point_map.fits | Bin 1581120 -> 100670400 bytes tests/data/small_sky/provenance_info.json | 27 ++-- .../Norder=1/Dir=0/Npix=44.parquet | Bin 7210 -> 7180 bytes .../Norder=1/Dir=0/Npix=45.parquet | Bin 6973 -> 6943 bytes .../Norder=1/Dir=0/Npix=46.parquet | Bin 7216 -> 7186 bytes .../Norder=1/Dir=0/Npix=47.parquet | Bin 6741 -> 6711 bytes tests/data/small_sky_order1/_common_metadata | Bin 4018 -> 3994 bytes tests/data/small_sky_order1/_metadata | Bin 8423 -> 8351 bytes .../small_sky_order1/provenance_info.json | 25 ++-- .../_common_metadata | Bin 2224 -> 2224 bytes .../data/small_sky_order1_id_index/_metadata | Bin 2649 -> 2645 bytes .../index/part.0.parquet | Bin 3867 -> 3867 bytes .../provenance_info.json | 21 +-- .../Norder=0/Dir=0/Npix=4.parquet | Bin 8255 -> 8199 bytes .../Norder=1/Dir=0/Npix=44.parquet | Bin 8434 -> 8378 bytes .../Norder=1/Dir=0/Npix=45.parquet | Bin 8476 -> 8420 bytes .../Norder=1/Dir=0/Npix=46.parquet | Bin 8378 -> 8322 bytes .../Norder=1/Dir=0/Npix=47.parquet | Bin 8344 -> 8288 bytes tests/data/small_sky_order1_margin/README.md | 29 ---- .../small_sky_order1_margin/_common_metadata | Bin 5194 -> 5144 bytes tests/data/small_sky_order1_margin/_metadata | Bin 12384 -> 12259 bytes .../provenance_info.json | 22 +-- .../Norder=0/Dir=0/Npix=4.parquet | Bin 11421 -> 11391 bytes .../Norder=1/Dir=0/Npix=47.parquet | Bin 133671 -> 133641 bytes .../Norder=2/Dir=0/Npix=176.parquet | Bin 28346 -> 28316 bytes .../Norder=2/Dir=0/Npix=177.parquet | Bin 86262 -> 86232 bytes .../Norder=2/Dir=0/Npix=178.parquet | Bin 92634 -> 92604 bytes .../Norder=2/Dir=0/Npix=179.parquet | Bin 99874 -> 99844 bytes .../Norder=2/Dir=0/Npix=180.parquet | Bin 42177 -> 42147 bytes .../Norder=2/Dir=0/Npix=181.parquet | Bin 54742 -> 54712 bytes .../Norder=2/Dir=0/Npix=182.parquet | Bin 72960 -> 72930 bytes .../Norder=2/Dir=0/Npix=183.parquet | Bin 67771 -> 67741 bytes .../Norder=2/Dir=0/Npix=184.parquet | Bin 80265 -> 80235 bytes .../Norder=2/Dir=0/Npix=185.parquet | Bin 162064 -> 162034 bytes .../Norder=2/Dir=0/Npix=186.parquet | Bin 31761 -> 31731 bytes .../Norder=2/Dir=0/Npix=187.parquet | Bin 43705 -> 43675 bytes tests/data/small_sky_source/_common_metadata | Bin 5525 -> 5501 bytes tests/data/small_sky_source/_metadata | Bin 28968 -> 28720 bytes tests/data/small_sky_source/point_map.fits | Bin 1581120 -> 100670400 bytes .../small_sky_source/provenance_info.json | 27 ++-- .../_common_metadata | Bin 2296 -> 2296 bytes .../small_sky_source_object_index/_metadata | Bin 2729 -> 2725 bytes .../index/part.0.parquet | Bin 4153 -> 4153 bytes .../provenance_info.json | 21 +-- .../_common_metadata | Bin 556 -> 556 bytes .../small_sky_to_small_sky_order1/_metadata | Bin 2929 -> 2913 bytes .../margin_cache/test_margin_catalog.py | 1 - tests/hats/catalog/test_catalog.py | 2 - .../hats/inspection/test_visualize_catalog.py | 1 + tests/hats/io/test_parquet_metadata.py | 3 - 57 files changed, 153 insertions(+), 165 deletions(-) delete mode 100644 tests/data/small_sky_order1_margin/README.md diff --git a/.github/workflows/build-documentation.yml b/.github/workflows/build-documentation.yml index 638e7b6e..20d31138 100644 --- a/.github/workflows/build-documentation.yml +++ b/.github/workflows/build-documentation.yml @@ -7,7 +7,7 @@ on: push: branches: [ main ] pull_request: - branches: [ main ] + branches: [ main, hats ] concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/pre-commit-ci.yml b/.github/workflows/pre-commit-ci.yml index 39663cc7..5e5854cc 100644 --- a/.github/workflows/pre-commit-ci.yml +++ b/.github/workflows/pre-commit-ci.yml @@ -7,7 +7,7 @@ on: push: branches: [ main ] pull_request: - branches: [ main ] + branches: [ main, hats ] jobs: pre-commit-ci: diff --git a/.github/workflows/testing-and-coverage.yml b/.github/workflows/testing-and-coverage.yml index 3d3b867f..b388807d 100644 --- a/.github/workflows/testing-and-coverage.yml +++ b/.github/workflows/testing-and-coverage.yml @@ -7,7 +7,7 @@ on: push: branches: [ main ] pull_request: - branches: [ main ] + branches: [ main, hats ] jobs: build: diff --git a/tests/data/generate_data.ipynb b/tests/data/generate_data.ipynb index 17a41e55..f926b499 100644 --- a/tests/data/generate_data.ipynb +++ b/tests/data/generate_data.ipynb @@ -36,7 +36,7 @@ "tmp_path = tempfile.TemporaryDirectory()\n", "tmp_dir = tmp_path.name\n", "\n", - "hats_import_dir = \"../../../hats-import/tests/hats_import/data/\"\n", + "hats_import_dir = \"../../../hipscat-import/tests/data/\"\n", "client = Client(n_workers=1, threads_per_worker=1, local_directory=tmp_dir)" ] }, @@ -60,14 +60,15 @@ "metadata": {}, "outputs": [], "source": [ - "args = ImportArguments(\n", - " input_path=Path(hats_import_dir) / \"small_sky\",\n", - " output_path=\".\",\n", - " file_reader=\"csv\",\n", - " output_artifact_name=\"small_sky\",\n", - " tmp_dir=tmp_dir,\n", - ")\n", - "runner.pipeline_with_client(args, client)" + "with tempfile.TemporaryDirectory() as pipeline_tmp:\n", + " args = ImportArguments(\n", + " input_path=Path(hats_import_dir) / \"small_sky\",\n", + " output_path=\".\",\n", + " file_reader=\"csv\",\n", + " output_artifact_name=\"small_sky\",\n", + " tmp_dir=pipeline_tmp,\n", + " )\n", + " runner.pipeline_with_client(args, client)" ] }, { @@ -95,15 +96,16 @@ "metadata": {}, "outputs": [], "source": [ - "args = ImportArguments(\n", - " input_path=Path(hats_import_dir) / \"small_sky\",\n", - " output_path=\".\",\n", - " file_reader=\"csv\",\n", - " output_artifact_name=\"small_sky_order1\",\n", - " constant_healpix_order=1,\n", - " tmp_dir=tmp_dir,\n", - ")\n", - "runner.pipeline_with_client(args, client)" + "with tempfile.TemporaryDirectory() as pipeline_tmp:\n", + " args = ImportArguments(\n", + " input_path=Path(hats_import_dir) / \"small_sky\",\n", + " output_path=\".\",\n", + " file_reader=\"csv\",\n", + " output_artifact_name=\"small_sky_order1\",\n", + " constant_healpix_order=1,\n", + " tmp_dir=pipeline_tmp,\n", + " )\n", + " runner.pipeline_with_client(args, client)" ] }, { @@ -123,16 +125,17 @@ "metadata": {}, "outputs": [], "source": [ - "args = IndexArguments(\n", - " input_catalog_path=\"small_sky\",\n", - " indexing_column=\"id\",\n", - " output_path=\".\",\n", - " output_artifact_name=\"small_sky_order1_id_index\",\n", - " include_healpix_29=False,\n", - " compute_partition_size=200_000,\n", - " tmp_dir=tmp_dir,\n", - ")\n", - "runner.pipeline_with_client(args, client)" + "with tempfile.TemporaryDirectory() as pipeline_tmp:\n", + " args = IndexArguments(\n", + " input_catalog_path=\"small_sky\",\n", + " indexing_column=\"id\",\n", + " output_path=\".\",\n", + " output_artifact_name=\"small_sky_order1_id_index\",\n", + " include_healpix_29=False,\n", + " compute_partition_size=200_000,\n", + " tmp_dir=pipeline_tmp,\n", + " )\n", + " runner.pipeline_with_client(args, client)" ] }, { @@ -161,14 +164,15 @@ "metadata": {}, "outputs": [], "source": [ - "margin_args = MarginCacheArguments(\n", - " margin_threshold=7200,\n", - " input_catalog_path=\"small_sky_order1\",\n", - " output_path=\".\",\n", - " output_artifact_name=\"small_sky_order1_margin\",\n", - " tmp_dir=tmp_dir,\n", - ")\n", - "runner.pipeline_with_client(args, client)" + "with tempfile.TemporaryDirectory() as pipeline_tmp:\n", + " args = MarginCacheArguments(\n", + " margin_threshold=7200,\n", + " input_catalog_path=\"small_sky_order1\",\n", + " output_path=\".\",\n", + " output_artifact_name=\"small_sky_order1_margin\",\n", + " tmp_dir=pipeline_tmp,\n", + " )\n", + " runner.pipeline_with_client(args, client)" ] }, { @@ -191,7 +195,7 @@ "outputs": [], "source": [ "import pandas as pd\n", - "from .catalog.association_catalog.partition_join_info import PartitionJoinInfo\n", + "from hats.catalog.association_catalog.partition_join_info import PartitionJoinInfo\n", "\n", "join_pixels = pd.DataFrame.from_dict(\n", " {\n", @@ -230,18 +234,19 @@ "metadata": {}, "outputs": [], "source": [ - "args = ImportArguments(\n", - " input_path=Path(hats_import_dir) / \"small_sky_source\",\n", - " output_path=\".\",\n", - " file_reader=\"csv\",\n", - " ra_column=\"source_ra\",\n", - " dec_column=\"source_dec\",\n", - " catalog_type=\"source\",\n", - " pixel_threshold=3000,\n", - " output_artifact_name=\"small_sky_source\",\n", - " tmp_dir=tmp_dir,\n", - ")\n", - "runner.pipeline_with_client(args, client)" + "with tempfile.TemporaryDirectory() as pipeline_tmp:\n", + " args = ImportArguments(\n", + " input_path=Path(hats_import_dir) / \"small_sky_source\",\n", + " output_path=\".\",\n", + " file_reader=\"csv\",\n", + " ra_column=\"source_ra\",\n", + " dec_column=\"source_dec\",\n", + " catalog_type=\"source\",\n", + " pixel_threshold=3000,\n", + " output_artifact_name=\"small_sky_source\",\n", + " tmp_dir=pipeline_tmp,\n", + " )\n", + " runner.pipeline_with_client(args, client)" ] }, { @@ -272,16 +277,17 @@ "metadata": {}, "outputs": [], "source": [ - "args = IndexArguments(\n", - " input_catalog_path=\"small_sky_source\",\n", - " indexing_column=\"object_id\",\n", - " output_path=\".\",\n", - " output_artifact_name=\"small_sky_source_object_index\",\n", - " include_healpix_29=False,\n", - " compute_partition_size=200_000,\n", - " tmp_dir=tmp_dir,\n", - ")\n", - "runner.pipeline_with_client(args, client)" + "with tempfile.TemporaryDirectory() as pipeline_tmp:\n", + " args = IndexArguments(\n", + " input_catalog_path=\"small_sky_source\",\n", + " indexing_column=\"object_id\",\n", + " output_path=\".\",\n", + " output_artifact_name=\"small_sky_source_object_index\",\n", + " include_healpix_29=False,\n", + " compute_partition_size=200_000,\n", + " tmp_dir=pipeline_tmp,\n", + " )\n", + " runner.pipeline_with_client(args, client)" ] }, { @@ -293,6 +299,13 @@ "client.close()\n", "tmp_path.cleanup()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -311,7 +324,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.14" + "version": "3.9.12" } }, "nbformat": 4, diff --git a/tests/data/small_sky/Norder=0/Dir=0/Npix=11.parquet b/tests/data/small_sky/Norder=0/Dir=0/Npix=11.parquet index 18ddf0c2f56f456a83fa59a92fd7732b1c04a274..70411a72b663050ccb2c895196c783d65009b8f6 100644 GIT binary patch delta 467 zcmdnsI>~iICqEB&d`47VJ8e%8y17^~o|PvjGQ6TZF(M<) zH!FMcKWPd6v_$u?j3f`If>bw)O8?-=iZYUu-^n^n&X9Lutem`7Ua#KKF%o2Dd1Sb0 zc4UMvkev<0&c%*_j*f|rKsUIQgWTf^;#UC~j$S~9qfd8ex49s7jx_LBfv7IdJj^31wMFPe&kd z1rd&sK#8R6^b=x;GuK~hm5F~X@T(#Xvq!YHRS%00|vv#(@4t3W}rhhJ7? zL{M5HPbi%Y4 zD4JFV^sNC%k+X?@nf_)A1s*0w+sU3va_nY$26_gPb0!xmN$|)pFfd#dVqgex3^D`& Dvu%eB diff --git a/tests/data/small_sky/_common_metadata b/tests/data/small_sky/_common_metadata index 31b324b6ad6b15b219ed178281981be2d15030cc..2fcde15aefa95fe40d20bddc139480f6df8031dc 100644 GIT binary patch delta 361 zcmdlaKTCeXBp&YgjMT)Og3O9|Bg=^kB^94>Rx4%ZrKDEGC+Fvs=H?YESt&#-At~B; zkDY}Ru4VE^77-+#AZt0J`Q%PkBhCfF`XEr{IQa;x9g|b}WEM7MUKax=M@L5xcJz;& z?82tV6BX`Ooa~-s5>e%8y1A4so|PvjGQ6TZF(M<)H!FJ*rv!gmqI+0Il7~}4s+&cn zfAC~}p81TGlh^a;)jK*yf=nxq3^&b=jPM1rvw_&T*fG%2G11Z0(b1(GWR)w3Uj<}1 zdI1@ZPWd3FKZpQHI68akm+9*R8BUHFATh^e5CL{0h~eny10o`fGLRkSm{z90Ie=G! mi4`R2I60GFj@?|(K+iyO&g6;w5+6HKuFha* zI)enEE(SAAfTUAeqC1-HK+&`^pnnZOikwaK%k(!p@bWM*+D;DSmjeb#0lyN*GJXbz I0LLIh03LOH{{R30 diff --git a/tests/data/small_sky/_metadata b/tests/data/small_sky/_metadata index 02a7f7b7baa02100dfea8e1f0eca2709d5385315..5f7248a6e79f3ff0a8e7268580143c5f18048658 100644 GIT binary patch delta 500 zcmeCuc&0vKl00{OMrvYCL1sm~k)^7P&l+I|AqEBpu{I`28Ob>ka+7y6DNMW?%Lrn~ zPYz~`hq5j+CO}ysOsQa2E0mwe>;`3>VD^EsOj&#(EQuesGNLS!rVMNXk`l;PPu|B8 zqWFxnS}8LxCAA_xIX|Z~H?LU9N+DVa$$^{ASlL^rE7YOTvK$YWU87?~}r}D`$T*|yI22PHSjv(ylA31prmm*J8 zxLa|udyYv&m8a?Evt044JUNl!73GN$8EL***^@u;O7N#8x`$;Xc{mlMx>;2E2T#`L zkDh!;z=g4LGP|H&y`y6!$jI`@aMSF_2wxyO8;G5Y9RnR56CGV09bL*n_PK)iRX~QL z7m(rTln-M1g9wm>qqC=enZ7=d;pCVB5_3!j5nzXc7>MrJ{Aa$-q*W?o8ag{q9t8es<^1_lPPHYQ0K$vG18`V%+SGJ#0> z$$pISP}W(-1SrdwDHY68n0y(^lAj#G>;{$H$Ls@TX|ec1SkgaiWkgvdO&Qn(Bqh+S zpS+$WMCCnawG!Ad@yYo)rMY>&QdN~rGKtisC9!j0te$p<-v(G3gaEN3*Ee3H|M zbB(Y*2vns{X5q49at@sA!lf+h=;;Uqt{}oO5{RA3Bg0LzBO`nr9sMmPmvAWxBt|$@ zMH;yoL>T3iM!AQXY+lF}&nj4u?BSOc84;A02vi;wk)ApEBA2MBvsYT8dro<>kzYoN zn`J>#Zct9D^JITs$;mzfZH$?dFAM0^I|l--@o{tl+EwfbWCwz6PY3c{Kr)WGAi}*I zL|T9dpoF8NXNb1GK8Wk;40fwCND%65Fw+D`I$>H46iq7wdfEV_$k{}{On-BaAP*Cx Y?c}+_a=>ueBdo;nPndxrz%j@W09~P=rT_o{ diff --git a/tests/data/small_sky/point_map.fits b/tests/data/small_sky/point_map.fits index 66769c9c47ed6828497944efc6fff3b4dbf2090e..241059f75ea9c3116cb436b49a44ae70e95d8fab 100644 GIT binary patch delta 7616 zcmeHMOK1~87^ZnNn>0<**ER*eMQtK=wy0APM}c}-sCTIPw7^DCUtH>2QGG(I zn+1*nb>UK1MRkK#g#{c1s@|f?P^y5Y>WVlDRJrwkRiSNbYJ4WJY3b&E`|ex4oB(wu z?f4PB5kiF;rRu(Jn;LVJ^wJH|jTfYyynX(ho(oc7C0&YGYlu~&!Z}TaXr%eseO$Nn zdsK+4Ae6pt$i6zOSAA3n5ACA+hz?sR($X2y!kZoTt!LCU=I!{Bp3*2VhXi%Pz0QXy zuJ0qwzawqPhRtKH)T`z9`kJw49_m(@=7vmc)lX^nJw3%>j7j$q!xvll`RoXcPmwH;{viBzjS_@R>X{9|Pb$-x6bgA}|k76FS`?rSal$3NbLTN$v zJaL#(twUOo%^HU&P2MK0#7VU}Y4MIrw~aVgl-kF;w5gjy)Rw_2Ue6Grm-h?wHs$2x zKkW+CS)*b8I<+E1(Ho!_v2%rn&NtHL5@|t3%I={w_m#BUaH$tCze!E)1ZnxHpw$5i zl0U`bP4!dIx=&h@O{U~1n{za%eiVzgEEjF*EOkY>ejAr5wPN|i zten=?arZodZF__UrY1IFL8CyETi5Z%<;}BLdq}naFH1T*!8p|^(&j;zdZ{dU+3_A{ zy>$N8N@;=qNuB;%dqDKVx<-pmfYix}88CXsM@jD2T!t1|c`mm5DD55~os;ufO1M(5 zb7a9Wl1HFn<4dtwtLW^T9v4Ak?mRF~YtlarGRf>e>Am?S~a1T#WHiUB)E69l+c zP7s1bPU4uH(g>w*ozreN#c*4vl6FFPl9 zDnXp32WMYuS^*iSd1GN9eVa&A11*g>K`AIB4X7Jez(UGH3qmWsdc0M!0?C%;UJ2}D z1!+ci7rF@1Lv+%{`KR(YDdr@f$*NKZMZj7}*WWT)e1}8tVVXb6DWMa9WK|@KSDrP& z#$+vy{Neo~zI%+rA#@HWYU4HcEiPz6uzoKWj)j) z9-i>jycn#|OVrYQ&rJVj_o_$z2UD>V&F@nzFEKH*1rML=azIA(nDcHShmA^*dr%Aw ztURX02pxWC%|jEaxdz{O$$zaOWFevpo5AvTv24^5jZ}Ox8uYo_fbU(b!6Jt4o1_mC n(a%BON6BKTk5U(Lj>y~a(|BAz8+@V5r*VFlE4bc=<=*K(UiKrn diff --git a/tests/data/small_sky/provenance_info.json b/tests/data/small_sky/provenance_info.json index a700a07f..9beaa898 100644 --- a/tests/data/small_sky/provenance_info.json +++ b/tests/data/small_sky/provenance_info.json @@ -5,37 +5,37 @@ "epoch": "J2000", "ra_column": "ra", "dec_column": "dec", - "version": "0.3.4.dev17+g922a4b7.d20240529", - "generation_date": "2024.05.29", + "version": "0.3.10.dev6+g5cb658f", + "generation_date": "2024.09.20", "tool_args": { - "tool_name": "hipscat_import", - "version": "0.3.3.dev10+gd573bcd", + "tool_name": "hats_import", + "version": "0.3.6.dev20+gc573604", "runtime_args": { "catalog_name": "small_sky", "output_path": ".", "output_artifact_name": "small_sky", - "tmp_dir": "/tmp/tmpgasth6x_", - "dask_tmp": "", + "tmp_dir": "/tmp/tmpcczp4su4", + "dask_tmp": null, "dask_n_workers": 1, "dask_threads_per_worker": 1, - "catalog_path": "./small_sky", - "tmp_path": "/tmp/tmpgasth6x_/small_sky/intermediate", + "catalog_path": "small_sky", + "tmp_path": "/tmp/tmpcczp4su4/small_sky/intermediate", "epoch": "J2000", "catalog_type": "object", - "input_path": "../../../hipscat-import/tests/hipscat_import/data/small_sky", + "input_path": "../../../hipscat-import/tests/data/small_sky", "input_paths": [ - "file:///home/delucchi/git/fits/tests/data/../../../hipscat-import/tests/hipscat_import/data/small_sky/catalog.csv" + "../../../hipscat-import/tests/data/small_sky/catalog.csv" ], "input_file_list": [], "ra_column": "ra", "dec_column": "dec", - "use_hipscat_index": false, + "use_healpix_29": false, "sort_columns": null, "constant_healpix_order": -1, "lowest_healpix_order": 0, - "highest_healpix_order": 7, + "highest_healpix_order": 10, "pixel_threshold": 1000000, - "mapping_healpix_order": 7, + "mapping_healpix_order": 10, "debug_stats_only": false, "file_reader_info": { "input_reader_type": "CsvReader", @@ -45,6 +45,7 @@ "column_names": null, "type_map": null, "parquet_kwargs": null, + "upath_kwargs": null, "kwargs": {} } } diff --git a/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=44.parquet b/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=44.parquet index 65ea6d912261208fb4ebc79e8c64b9bc0d68e6e6..845e3d923e8f0eb11a2b87db8009493e4d51b6b2 100644 GIT binary patch delta 416 zcmZ2w(POcJpOc3>J|i_Try#Q;-pF$D10Ji%5uAM}f}6i`-eE!!0V)epe8yR=l$n>3 zS`nX|pHrHfSFB{E5Uqq{*yb=^b`~|b*-APJN@r5t3RD~MkOWH@>O8IDf*Af`Wv z07*DHd+L|z>jN21ju{{^$7B!zb|{G9=;#9?B8)PSUFVoqroZ`wqy!TyNYZigLuoli R^GPyF9P5P`7y=xF3;}35f$snS delta 498 zcmeCNSY@$+pOc?2J|nZBI61K-J~J;RwPLa~=LvMN&F?wyFrfSixqb{1`9dz5q(l+rR&b5i2-5_405T2OUN{wv@%d5@qG zquJ#5f<~Ncg!Mt7Dt)q+kR6kA;N&bJWm!j0M<8$o5sr~S>{K2ZZkin#;p^z=Z!vk6 zkfK0hgi}?dk()t;QBG-;dzi`Q<3jPQ)&UVN$SVkWeT{EGjHbEzwObD9}wTD#|ZcC`&CW&dkqKFf^O| QNLq>Gln?_$fMbv$0P{tl#sB~S diff --git a/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=45.parquet b/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=45.parquet index 3621116a2a2abf0184c76785dbb84f2715445de4..d30d03c559e9f7ed03ec726c4e5629010ad4a327 100644 GIT binary patch delta 453 zcmdmMHs5T65(f`=d`43 zS`nX|pHrHfSFB{E5Uqq{*ycp1eUI znbCZ*h@cVY0%3g+sB)a_FKEZ)R6e<1P?^`oz{%0k5riH6BPX91ROE>YcPmbI&oPOp z@-*Ef6wk_&6B%Ano*0pl=9`s0IbU3wCoCh$!>J(E&7#sjc=9?C$;s7XPLsciyD(Nx zwvy1RcXW&d8Cf0~Zkin#;R|GE1F>_lW1ypBqNA&$qf0r+K35RG3dnHu0x}$(@6AUn@7txSLOZ3zh`Es&%mgM>ms zVo_mfYKd-gL4j^!QBi)mLRo52ab|v=f}y#dfu4cnoXPUiqLcNc**Vq=F)#!;1{neX DzRHVp delta 446 zcmbPlw%2Tf5(htDd`4zLadKiwd}dxsYQ~$95}gFP+8W|(-8<bi%Y4D4JFV^sNC%k+X?@nf~UZ58k!DKjr6 zwIV(_Kc_S|uUN@SAzBH^u+1^N>?~?H zjdBk&i7@gjOY#USNy_uh^v`rk^31W!j|z7yPIk{RiKy~4-TYZ7o|PvjGQ6TZF(M<) zH!FK`ou~wVTB3VcMv{k9L8_ZYrGN0`g`$#^^TeGdzmRZYtemVXsaNml7zwhnJTlxg zJ2Ju-$j%00=VHe|N5@1*pc`DuLGEz{@vDFgM=v15(J3Fq^al|j2}fs7{W5)hAj8Qq z10?2{3?jfD0WlmMeLzHnQ3kRL9n;G6H=mP~U}6PHI!=BrEyr%IXP{>wIcGAzj0BGi P0|Uc)AqIv3#~?!h|GSaN delta 428 zcmbPavB6@4C?`K(d`4zLadKiwd}dxsYQ!QkHe}bOZud z5aAdJ#7^ar;ilP<5x$O&{uYxL2q_9AMmSYP8o3!n80C~kxrdo-J}VT@Do~K@;g=N| z5tNn)R2~(PKKX;Fgm7A-dro<>kzYoNn`J>#Zct9D^W=Dm`HY#9|4ZoAI|l;I_i=Or zT2kx?WCwz+O$YK_Kr)WGAi}*IL|T9dpoF8NXNb1GK8Wk;40fb5ND%5?Fw+D`I$>H2 z6iq7w`qluX$k{}{On>tZNggIf+sP-T<=D;i4D<{n=S==6Ex{whz`$@yh=C!%F~|@A DulR+* diff --git a/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=47.parquet b/tests/data/small_sky_order1/Norder=1/Dir=0/Npix=47.parquet index e04d97c84ced32c4777d466e7b0c71bf9ca66717..c2cb304e86de741cb5a4309d4ead321c62122044 100644 GIT binary patch delta 418 zcmca=vfX5Z6*~`id`4i|&Fpa1*vWpmHxq#t3;zG^NPDL zR!$BQ*Q1hbFpKfqhq3@tD~b!ImijFAbu5);phcqI6CEn znEoIFB;n}nsb8kA4`etwW`M*TlR*U7?I4DuqYsFPFv>u7uwz=8{^rl(5=^WhNyo`N VQgXoX(3Vo-STDrD5a1YO2mn?vf8hWC delta 456 zcmdmPa@Aym6+1s)d`4zLadKiwd}dxsYQgG~zb`~9Edz5q(l+rR&b5i2-5_405TF`V%-pA)U`8mH4 zquFFB0VB>e!ulXkl|DICz>di|aPm|EWm!j0M<8$o5sr~S>{K2ZZkin#;p^z=Z!!6( zfTBQRgi}?dk()t;QBG-;dzi`QPXh6*0tLw)ep!(bL1~FVlSY0v|^wpe@CYKz1P5-gF?}1tjB`3nJXh zL8JwU07^JIdWLB0>w~zi&R|zMg9M=t1~W~7q*Gd=JDTM{(X=w4cMU*_oK5u0^f%uX b=V4;Bo%~Kx4j3Z5Qc4`BgcukC9D@u2QM-^T diff --git a/tests/data/small_sky_order1/_common_metadata b/tests/data/small_sky_order1/_common_metadata index 31b324b6ad6b15b219ed178281981be2d15030cc..2fcde15aefa95fe40d20bddc139480f6df8031dc 100644 GIT binary patch delta 361 zcmdlaKTCeXBp&YgjMT)Og3O9|Bg=^kB^94>Rx4%ZrKDEGC+Fvs=H?YESt&#-At~B; zkDY}Ru4VE^77-+#AZt0J`Q%PkBhCfF`XEr{IQa;x9g|b}WEM7MUKax=M@L5xcJz;& z?82tV6BX`Ooa~-s5>e%8y1A4so|PvjGQ6TZF(M<)H!FJ*rv!gmqI+0Il7~}4s+&cn zfAC~}p81TGlh^a;)jK*yf=nxq3^&b=jPM1rvw_&T*fG%2G11Z0(b1(GWR)w3Uj<}1 zdI1@ZPWd3FKZpQHI68akm+9*R8BUHFATh^e5CL{0h~eny10o`fGLRkSm{z90Ie=G! mi4`R2I60GFj@?|(K+iyO&g6;w5+6HKuFha* zI)enEE(SAAfTUAeqC1-HK+&`^pnnZOikwaK%k(!p@bWM*+D;DSmjeb#0lyN*GJXbz I0LLIh03LOH{{R30 diff --git a/tests/data/small_sky_order1/_metadata b/tests/data/small_sky_order1/_metadata index 810fded31d4ef3b3cefa4cc8cbb396f3f3f16cd4..93f6b2cd5cdacceaf139e98eee739f1000404104 100644 GIT binary patch delta 903 zcmZ{iOKcKR6o!X448yAg7z|}7Rad5?L}Kg0czCq7(oir8C`zz}OamQB|T^ug3HP|3Wa)4HyB ztXuL1dSf|npebv?K##0~V6=rVtP2OnLkEu&HfDurXk%80;&x_*XxGlH5KTH-dY7WE zc=S1$6}olH$*d54bN*w+;?k^0xaQJqNch90S&*>9YvU#S&TFIZ><^Ogms?9OVO`M1 zN%&UKMoHq%;aJETO>=gCRQxOXl7}Uiy*$41u;j8=IY>bJFTE@^(X@}HCff6{)I?c7 z`w2t;MMH;BfTdP%iJxNsvGXKQHhsaLfrv|gGo#>`!+KQ5mnR;m0v1#OnRSvyDs%wZd3j5gRgnRO_F$RH pzEKhY1uE5)Vsr=XEqQb|E}Ct02)LVwtKs(&Nhc;aZVAdK{s2Zu|9AiZ delta 880 zcmZ{i&2JJx7>9Saz(Ron622@gE9k+XED+k%CK`touoYTCQ-MSY@>MD{B-LW7jnNn{ zno1pyO?on3dTBWI(7!;VCodYW>cPbH=)uH!-)xiG#!2SQGrxH=&rEhV$#>Fw6&Nbb zH#|<3AR+a{jD1Yct88P~3&VWk5j|Nl%W(H9S_VZLgP8t!^bcGha=f>)dYV zS~Z3K^ltJ^`eG#XNXfGE(a;ln6nu4kZUAn7B+zyL=Ko?HwJ0gge20OZLY AkpKVy delta 122 zcmcaAa#LhNGLxwOWCcdiiIpxe)@2_ED+wyFc?KgBGo#t$*(~!U9X%a^z!gL|Mgp-@ yd1Sb0c4UO_W+&DxX2#6PYdMTJf8g+BlrhsY&@+(qkzrtfBL-hC28IB~AVUD#5gm*G diff --git a/tests/data/small_sky_order1_id_index/index/part.0.parquet b/tests/data/small_sky_order1_id_index/index/part.0.parquet index 124f8c89b614c6198fa4f7ccd1ac1176ee27a6ec..aea68416647143f6bc43edd675b12e458c5c9a97 100644 GIT binary patch delta 58 zcmbO&H(PE)0w<&S5h(0j*gopxU!fTDy diff --git a/tests/data/small_sky_order1_id_index/provenance_info.json b/tests/data/small_sky_order1_id_index/provenance_info.json index 676fe968..add01c76 100644 --- a/tests/data/small_sky_order1_id_index/provenance_info.json +++ b/tests/data/small_sky_order1_id_index/provenance_info.json @@ -5,26 +5,27 @@ "primary_catalog": "small_sky", "indexing_column": "id", "extra_columns": [], - "version": "0.3.4.dev17+g922a4b7.d20240529", - "generation_date": "2024.05.29", + "version": "0.3.10.dev6+g5cb658f", + "generation_date": "2024.09.20", "tool_args": { - "tool_name": "hipscat_import", - "version": "0.3.3.dev10+gd573bcd", + "tool_name": "hats_import", + "version": "0.3.6.dev20+gc573604", "runtime_args": { "catalog_name": "small_sky_order1_id_index", "output_path": ".", "output_artifact_name": "small_sky_order1_id_index", - "tmp_dir": "/tmp/tmpgasth6x_", - "dask_tmp": "", + "tmp_dir": "/tmp/tmp0nyfm4am", + "dask_tmp": null, "dask_n_workers": 1, "dask_threads_per_worker": 1, - "catalog_path": "./small_sky_order1_id_index", - "tmp_path": "/tmp/tmpgasth6x_/small_sky_order1_id_index/intermediate", + "catalog_path": "small_sky_order1_id_index", + "tmp_path": "/tmp/tmp0nyfm4am/small_sky_order1_id_index/intermediate", "input_catalog_path": "small_sky", "indexing_column": "id", "extra_columns": [], - "include_hipscat_index": false, - "include_order_pixel": true + "include_healpix_29": false, + "include_order_pixel": true, + "include_radec": false } } } diff --git a/tests/data/small_sky_order1_margin/Norder=0/Dir=0/Npix=4.parquet b/tests/data/small_sky_order1_margin/Norder=0/Dir=0/Npix=4.parquet index 966825609cbae39c17acd4ce83268e694da5fb0b..e125bd872a5b7f4d2dde62479c3e7e213c26fa16 100644 GIT binary patch delta 1222 zcma)5OK(z95H3n-O>gT1O$mYWXeB1Jxk7DPYHB){#|0_^t)V~@DO~8Kyu>z@f+kH| z=uTtK#+7^H#z^AEq`EdS+PE<;P5cLq3**9>3$~D0)3X?6?)m1M`Q{Ao<{!1?l=g&TeP{sLQ7n$pBhW=FTL%9o_*5j z*k`qs#roz{&)GB1s(xd%J!HF_o&D&0+{M~_&_8jq>kHPo+FfJpALQpw*t(eAmObvC zb-j|!ZKtH$rLA&h(>iNjv33Y(UL5RG+Ya+I-X8LX7{z%Q~~x@J2oiDf(#%49-o)})>^pavTsuoOIPBJz*KYM ze5#i#C6>2q>3DuER4B5@uvn7LL zN2Icn=-Wz)z6QD>if=*yt?UNyNDu|6f)_xnQHz%;c2j%y%y)mI$>mn4B%OP;wZ8so$qUiB>DZn;+Y0~dVWhJ|%GA4yV=^f_gdUWP$dgBh;Hx1+8k2Bl*jk=zm2rN4O E029w`SO5S3 delta 1335 zcmb7DOHWf#5T;P9#3E>Yw3hNH*l0jnYN?b2!`$AsfKp3AdT))^a{IIoN_fQz_umQAMD_HMjC&TgtlRA1ScwnuZ%X!&@m!^xgB z^|SHDRyL~{Q$1rF8ngPX$?}lB*1V~wtlT|#uBm<#>x?!c|E@xPe^ofsXxT&E&#GE| zUsbzq=2|1lVs_(fG zJwEbtzrdo($MC0UmZylcW?!ntIEdL>x%ehK8Od^3<6k#o5Z{YTjAan#s(FrxY)dD%xEiTEP zs-Hf<#04aA(J`zB&?o&flU$pR@1h&#&K99(xEPf#-2LIBhGje0r9(sn#EUAsNL%U$jP<@$nIxhV20 z#>fr+9pU1?p9qz93MIM5Q|mlg>zotd7_dBM$@F&d55^w|&t}&5HFRMHZDe6Old0ws39g8S7@7-nrdeG*g~y9Ybc-zfrT!G1!LgcBfO!idrO`h>;s=H&Sw_#O>ZXG2F{Y7 zhE?4|vTStdpV*vV$d=JX9uMvJlxxonUpmpVLmOP_Ek4!n{I~i@zq6;b&;FUa=h&>| z^7XWIN7*e`b{q?~TMiBN_JPr$cXpLHF4`O$n|URdu4Kx23f5wY{Y;G99yvTdJU~8- zdUSK-aMUw=VCHc(Wx|`T>O!PqI&TzYK?V?ju3&(?Hrct
V)m6=2{G~51aN)K>K za=l*5BnlfV#S)23Omgu;c1ft_gY%8Z8rdHkC#`b=c{+Z&{e2>+I~IIT*C2xycT2>^ zk{H}giNPjyS1`T_A!=nGfM0?bKotT2Vhu@N#@L7P4N4n)C2Amx#*Z-`0C=O*JYSe7Ca6MyP|-ymSf*MV8BsIWm_XU^JR^O*Nd0*V4fvQo zys?f{bW~}Assh91BPO$dDGB1aW)y&<2&`5m`ILXH+Yp-kBdfKQq-TPLG>v4uniziR zLL&fXF{glgAfaLJ2_1kinv@tXCOq+mRQ5J7jznn^wg#0@?oX_}v6hA>o77Ucpe+Jo zlUBjGfWYRoTpuNw?jbo$Ec?Ys54Mk08wmQODe1Rs`6w< zIR(l%h8TT|AfjkL=2F?+ZdxT8A#7-(g7212vWEHed9rb`3XC!=F7}yi)y(!5d0`u) RfA4#nYshNu>x;sw`#040dN=?8 delta 1242 zcmbVLO-~bH5N?rDNI+|;QfO&Q!CoY^)Pj}+X4-8FRVoyuyLhN&>6f&ngpXQ@;bhc= zgUKX%@Z!OXHxoD+e}FL@O}u!(e?W`}PwLFuLZF;T zxqNv|O|2*MrKR*keBQF9-9eqczj@E%+QC76S^E(W+xv81dR=dO@q4owZ|W}Lp+2sC zhQU^+?v2BBU`K1K8NcdG__B4Uk#cvaZRlKMlKPA_5&wxs{Y|d3os3Pcvu%us9p-_9 z_ARH-oK#l|8^zL^bIftid3v1P{SJNe;K>1}DGV>@tmOqd0FG=!Hoj3$X)*eu0XT7+F^2DS!yzQ&XQI zB?lfU-mpjZ7B&_l^S<3ZQ&MLrs}r#YiX6|R2)-a^(cWXZSg5GpSZ*ooFE1A3g>-Nd z|CqXW@AtgY{sSI8-GEQ5k~WW@t;3Tbv9vG*$xlHRz{hHDWdp1gW&rL8^%Z)SRECz> zMJnul?5F@RE&0)G>$k2eapTZ z8;T2ZvLCe>fY8XMO7B>$^(0YQCZ>Oyn_Wn9)52}8Wr_pK3yD}}L5@}zuL(9Eb>PJS z)D(HJ-EQ}Yl3EO(M8e{c8oB~(t4Gf>Q~27}V~NBIN<|Vch^j$|Nkn~v!jOax+vPDP z4OErdx@t1BHJJiVnu(V{3g0HW^x|6(Xf)U@E-_0g!HF!di;!w0e+xdGuoNaj{*0v|ZEF;cgQR2=DmEzSF%s zT{b(8noaH<=sM?gt*qus%juh$a?Um5bnSICcS7O!PyReOUE*K*eZmyCTYbG-Hj!3c z<4Nl!APc{#qN_SaG$))vbCXrv6FBJLk z8QZayj2zErXIUi|nyy6``J=&W&EErY;ULT>LXD3Pc^fhvsmh_-DLGWfbe`~42xHU& z06`TLfC@+ea+TT}CmbMr1sjGy5d&l?1PRjyfIl`Nid9UMFtPz=DNtc+1!zz9=GAgz zxsomUqhu7KYaUVb>y$`r>i+_CmM);zOPK^V%+_$KMt%IBda|>(fgtrl^^UU9Kz1Th zOpyq%C|wR(2M8BD{Q1zxjrKV!4BHe%-Oc3!=F>=(0=jX!#MTTbF^*)0J%V$)El%#X z$bUHo`lW@6ss$H!q@gbIvv#{XiDfLh+aDs0u1BF?Tof4amkviGLO3PI3khGmrk4FR z!qFH?(!3!=0ha&b{qBZHvyuE>4Up?7LOy}%zUK-pECmhU^tEAJKCz5<#EPkr^jwPbSMT3-ak~vZhrs( delta 1274 zcma)5%}*0i5N89F5?e)Rv|@o$K4MbCLM>|5D6`$R6s4AebhjLuvd|X$K?xsg#Weoj33IW`4hUyWe3~`nYepaygk@ zDXnX%jpRykY55g9G;IsVY@}_V^{r$0_$K>&$;Aq-9c;JR&i2}dg!io5+TXI@JAA^P zSl#TJd0zO+3T9W!59jb9W9Cnd6{|}7NONP7?N}Z*v>B)`eY?J_n;lv0gWoLME>1J4 zt>!li#dX&M=Yp$VLG{Gi)?%sCwAN*-PwaJGXT__g>O|))p?>FIP=-+UrOPASWhR@a z{g+)N8v_Qm$HEkQV;k&&FrWpPRd8sO~bPC}KCBp=}y(^sx<#o*y$u0$brNu%lzwED;x(|f^Mze_<3+#hK z7FO6#$5&3dh4N)g#A#$46_nkgcpp;%Kc5$|s*R3^ zqv+4i&lP-SoEO!AS?nb0C;PkS!(hjLQJu_|G9?zan2nZXBcAFqRAN^1M~s3hEj~7Fr`Cz7_3nW#*nyk zV@$6*W9qUC7be{q6J5G6#_sBmFzL>XCZ02cEi|doyEt>cx#xWM`_8@Z;0e54>@l26 z=X04-Wpg|2_OLrWkGMVdt!KRJtC`>1(O0-bc27U1KeF&2EtY9^)ahWayNv8We}j9$ z`g`r&Z>;>!QRX#XVQan5+Q{~Q^^acSu3&&w2geMlmEe6@DqT<3!&CcvgXwNVX?110nn~t2 z779i7!93i*k&zPlY)Gu+g0r>gGILsP@4qr9xPPhK*K&q0w3Bo?Yizpa;MGTOo;3rHQ0G`;SAXG7lhp`$UM1dNc%SU^X zD=(KD>y>QD6D6Y%-Es(mM=goOX8uk2ge;|`Y!QvK9RqIcj>L)v`0{oSn%x`7D+Oe5!G}#O4er(T)@~dj#iB zBu<@3WG}5|Q(&ngEB@tUYN!kB)5s{lifv-_L?0rBt|v}kTogoL^HyskLO3NQ3Q2dO zCYL=m!qFH~(!3!^0oMQEv+jn9LXmtc2S{}!A)g?szUK-}9Pn$t>2E{3e5%skkyuQv zEP(4_Q-I=+VHp?r?|a9X*F3s6n*ySJ?*796y76xsSZs0jbSFvPHRTS&OZKG;I%e%nQ|Ag}}5 zZDEGFhelm64QL8E(C{ZUw9G_VQxgDjV6#KR19???qJ?4sHB{Nk#vg=R&xUfQfx14G ze59%ALLT8+waDIGu-gjxc%`9-lBMNXq`p*5S5~5}e#a-_ztL>pzQ7VrSy*K+oxy34 zNmdM^=o3%`2$Rw~MTvC69KbAB-oUl26SPh)-XI?)hX(*DIl>NIBhF1tovESR!hB7t zuYgHB{>CBf!*kB#F>5Y^5-!YUZZ7f;eq1QsGC&>r*Bgd6o0S zItd_sAeTD?+^1cRrvL%uc#X-sCkf@0$lkbImn5Q7Cbo5)p>9xP?jqgCe=LujB8%8p z_f8pOshrIu3t4rhxpbAks4#L;{QySdz|s1+AADSGa`+{^8f)i(r4-)c(~NPd_2czi z!k7r{?zHs-2pmsWw1&**hN~zfWn{U5p(68lj!%accoR66^-c; zq|vLx4})Jm9)ffRO6;x2F`UoPR*@QuSl#Fmf$26GyX?J)U*fEH+>*9;b)_I3{0p&s BZh`;+ diff --git a/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=47.parquet b/tests/data/small_sky_order1_margin/Norder=1/Dir=0/Npix=47.parquet index 9cad481dbbcf455bfeb311901e38784ff90a2911..96ec2bf8fa4b838abc2f215e3a5e253c3877bfbb 100644 GIT binary patch delta 1208 zcmaJ3AyT$rfo-*BM`CMIrl=fZ{Vjq#kBYFiSxi!B@7((ozFvPj(ZOGc zr?wN>{BAMsaIt=Vk^N}{?u5PR7;pbQXge8X3Bx}3kUi(e+MkWsehjmphG}-(xyF59 zzj&+RyV>@N_4iD%rLMjq z=n%k{o?+M4#1SqjIai5r zAc&H50_f*%{ljNn3{jW;sYVTu%P2zbLDZ(PlCEKoq)&tXR`i=Emi~yuOl)Hn?00Jo zs8R?s0gECOHIhP6d+Qp+M}F91T4YafbVa@&*MRR3@ delta 1260 zcmb7DOK(z95SEsfYN(>rNNAyyQetcg6sbyMjB~j_X)Oh;axYCJx3|3HQEDHEjcL+l zcjhGO&cwA*6IL$jZ*ZfFuDW(%8sox57tS2ohT^J|Fw8k;=9}-EbMF4#*YMHNec^mM zU#PCC%38WmS;{f8oht>I>8G1goBR0PFxH{W6vCU)$!ngEsv|(TCaGn4jW&U21V_&~% z=S0)$a%sI>S+!5v7VIZAG@kV8v}3KR>|-{2wvZ_;r7KD~gC%Vh?8T)xJGl7kv_(NS zXY^`rux+FF>~E8sY#i9G@s4Ji{WUr)5COCxIgs%u)a2wuQI=)_!oXa8Lk2|>p2^;j zNAi}|Q{jicMyfB}WvHrC(Z{ls$SRngl5*_%6_YWmgiAHm8_h3;{ME&BqLc}+FZ~Pb z!({{eV?NhV%(peCseGd;u@k~x2z7PkP;|glH7F9~Ek_*?! z`^ez|Kuq+r&z9kVHCdXjAnojYMW|*#EW0<{ZUg}ch-}v~XkW%qIL=ChOLg2x(&%_r zNE~Gqt98qBB5l6xuj66y@zXJ^ z$1o-}@d#%Srp-qr{Lr|*;6zQ4I6kZp0pbT@v5mk3+U5BaU;;U=F?#Q~p%`i4>3tUK(n#uaD6ln zeqAkec#&ETwPL_x46FEUhB(y9(K80UE!HbUu|DOo9GpHX@l60P<$zq|}91>L$ X6dBXmM(~q#*+zR3rZZ<^kQo05%*lWu diff --git a/tests/data/small_sky_order1_margin/README.md b/tests/data/small_sky_order1_margin/README.md deleted file mode 100644 index 3487c60c..00000000 --- a/tests/data/small_sky_order1_margin/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# Catalog description - -This catalog exists as an margin cache of the small_sky_order1 table, -allowing spatial operations to be performed efficiently and accurately. - -This catalog was generated using the following snippet: - -``` -from hats_import.margin_cache.margin_cache_arguments import MarginCacheArguments -from hats_import.margin_cache import generate_margin_cache - -margin_args = MarginCacheArguments( - margin_threshold=7200, - input_catalog_path="data/small_sky_order1", - output_path="data/", - output_artifact_name="small_sky_order1_margin" -) - - -if __name__ == "__main__": - generate_margin_cache(margin_args, client) -``` - -NB: - -- The setting `margin_threshold` at 7200 arcseconds (2 degrees) is much higher than - a usual margin cache would be generated at, but is used because the small sky test - dataset is sparse. -- The `small_sky_order1` catalog only contains points in Norder1, Npix=[44, 45, 46, 47], but the margin catalog also contains points in Norder0, Npix=4 due to negative pixel margins. diff --git a/tests/data/small_sky_order1_margin/_common_metadata b/tests/data/small_sky_order1_margin/_common_metadata index d27a9c833f150b3fd3a2d2faacd4ae00d7f34a95..e1de7b96777565d9bfd817a339aa76c0449ebb37 100644 GIT binary patch delta 1098 zcmb7CO>YuW6lId)w2^*jo3u5?Qq-mi$^f+~bYXHI0|T@;Olv4$BEmq2@?l7g^08s* zw)!^ijK6`UiAxv8jk_*f_y=7054`sc0-Dt5Tim&4-nsXkbKm>4`DXL%wd-=BS}8VK zC!Int{OVm@`od?W<(bj1fnSoRHb&5M$L%>jsWi%kr^RN)v+MRul=E-SWqvlb%auzq zfA9SMe`P+tDhH(C>pkRp9F+hoeN|-hP!s#tV zX=7p}kPQ&0K!w8`V$!Nek7lHEjWVGrlJTp`!&o%NR38> zak|9e3@EXWWVStmdt-^)V~O0mcx^Fy&@wIcaMT)l3eR4-v5~_vrjPX@vgmpY`r@L% zfPcNZ)=v`7YnfU$nCY6$aF?*2#+Ed1h*5y@FMQG65GkIlPWAw;hfT;QFyVWzHpEfY z_Dz5L_T>}Hen+gHKT1L1#ZZ8y8ki}#$)F=hCPne+1;lrS%WR64Lz}swP28}^vq2;c zS)dXbi)}XMp$i;|5ory2sW`VbHNXcch9_DW{Kw*LdB^AX`By)4_tLuSy2IgcEjb)% HI0*a%+NR@ucwvR!_bh0?UJqKW>M%RDZHM}J+C$wI)p(u!-1zmpbbAPs0qa-ic+vGqc{|@Ib#}xi;oVgl2;0~pM5iu1K=rd3Qh!EF1HHDbMI>0@_-NrRD z3ECl-vdKrt;Q>I7#dyzi_41yst+kPGb)&6xD!^KDPnNL=K!WkEXU?~cp>l-Er>s71 z6liDyrLslt_s+fuo`7w%VtpJ8LE%0Ylx^e*i6j*#Ak^uE$RE<7fS99c;ri)G0<7M` zA~8CKZVqEovz`_VQndM)gddvFmyFtmChEfu5g>lRnx_anpk0Bd00HEL#_ZjrhAb!Z zkY^^KP`PBZXW~VgPHF62qkQ~FdE}Bz#<|Y=n;0w2Vj){CYHR(?o8nDHuqNpTa2f~s zmUpfqj?9*N90{8_go-;;MZ(UX^?JscyV(6J8VH-pTe{7}<=`p-S%yV!W0++8x#wb> zIw^hK$#S^Y9PR>+La-Tvbb3kn+{ITQ(7f%P_6vbVfi`M1Dny!&YJyTO8UHYGOq!&mDIBBYmpFCqbq&yrF#~5P+#xt@ z&zv)JzB7X#o%=od@nu7CG?j>@vlH{tR@WzA#X0Nc{5ojhMwqz4okj5ubJz06n6U3l zFmZFAkBMSU4-*$Py^`QRtg#q1rnbC-aucsU1pd_=lOnh^7}h#9TfDFam$VIR?lJtJ zwd?*i3wOg%E&SykzKCDpjA6>vo_ACtdgfrz%&P+mP>eTfrp*9t4O@Se5-OW-yj zQEb=!Xcewo<#MIcxMkQZc4)t_30w8>$kGT04Q0|HMhF>_Ii-h9!&#LZEMdiClpY?G zFaYL8N*M%mk4qT{bDd=jhPiLc7?9-nHS-?V@JcxYl)|`8_`KYyy<`zK6*xBk0Tfi& zbvNpSr)og9_A@M0I5bae!Y%l+LfI@+sfT|mMr5i?Wh$ri@K>e6hf#A?3Lm=dDuoZ- z@_vQy*Zr@nb~t5J*k}~LY}$kEp;1*1w+gOmr}my%ST)NRX9Su~nQvCxbx$qA_ZHRV zv9(XZRO8Uzsu#BF;I*2gatWop9CNJW*n1pN2SGg{D=xiu)C*4Ux+=PC9A6Ogg zhRC4*d_0hTd1$e#ZT&<;RBK2N56;iUhEn5wlT$GB0$(;B3k;-2JKfoYzhfcT58oV^ zTJJyT=bk6>G>A=JZUNqGYFP^4vGIWaLL}f{q_PkFDbhv9q=yi1oP-I<5+5OfIV`am zy$AgXS~T%Y(SeNO;6;xMgt$V7MRAUba2Jgxq!R~Bls^xy1zJ+^>EW^LXxbG-SQ4CV z6h)U5(j995{|)dgX#*+W7aO7lyJu-5TG|jE5kGdq@LKy$j4+l5FG)bgzj>K`09u5jZx^ajE1}2AE2NvSf zt_AeL5KY2{6F&}QO!4Z=SyJB)Ms;?l)SHmNBHaaBK-^MGdM0J>^Rkvmvy#Lh_K@6D zk-jL)Jl>X#&CY0&cWs0zW_RoZMtgJBD?s@mU_+tpW;Prd% zIrrT2x##Bn65Tj+{id-XIx;$SE*?7{9i12)x_IN$IB)mD+&()vxzn2bgzML&g^z0T zLw-<`)_p@vrgw9?7R@=PYh}`5D-}z0*7}T0lhv~8VA#|hQ#bJUt*{^n2Ht$pe3_Fr zEYf`oaCzmNQWy6t+|#@Cmj&qu8~j*&h^_KVc1Vg=IG&lUdr&6LLrdl{-r)r<+X1&T z-_oQft6!6!XGNKOS6f-i9@6B&oPZ`5a#}R`BBxcA!iIHkl6P{oqxpGJAu%zMdxA7C zz_i|@X5k({AkU?LR3x`$6@w_KW>svfV z2e$xEOfLOxyYzgE$LQdHW;})FTb%hzPI~5)R$y|!dPyblaQ`qPN0|%3LbXTC{I*c- z6Epoqj4hj)-xsMpV`giy+Bat2EmnKSOtD1mA2Y9(7BW9QWxhQ=nZuL-c@gF;}kNF1kbUI;GzEm3+VY==hIV;B?Nx=p*Z3^zo7`G(CUxQ|{uO3!k}4;i0n;HeFe8$@SY#=N{2j@RPfiI|6!7ZO&g7FOC`l zCp@ol%`oRFFC$HajsQ+1LjQcJNKgG(B-}%M=UyE=6jC)#2LYOmFDTy45vL zRBxJ!pKL!H33m<0Xn1cpar0$M(Qqu3OvP*4M+SopQv>5&$szyhMAvXZ2 z^oZvp;pPci>~QylZ)%9h+bceYiK=JUeh$c~5mTC-M`*4z7m! zD#x7SijN1kN0NaK8mjaxsBDpiA3xsZ=2yTuP4q-tN0V)e(yf~R(LYxz> zb+eJgCsqV^vVoik67Qm{>*T3&MvUf@PskZ@9+1)p0qyk3%<^_jj_H$~<&}9|(w)*B z9i1G3z|VDZv%DftRJdDlvU`q6M3txM<}LE?*p0$6l02LWQr#>n{ewM=J)OOB6T`!b zqueY@Bf|}IJhSrCJ##Gck@Zc!qduR}cyf(~LA|46B+zW9^2l)0?8pdTAlm@UcLZAN z?da*~7!U}w-Nn-}(9to^(bdt>y&U8k0}$U4#P=-+($1kEULuHa0cr$GI64{z80+hU zqys&{sy#q55RG7#bGf6SqoW(D+cq0$iZio<JK7IeYMo5h1vZt+$tf%U~F z=jW8><`pYhDMTxw8L1}miIoT0lF5p4BB%l%C4?vE${I~>mNQ~Bo4ifVi1Un;J{UMo z{wQb1)wXEd7Zr(sa<5(qTj zBhVR07CWXpI%APyY(gIEzp83oB6ep!(bL1~HMrmWdf5$TSb`836u8Eq$P>BzC0 e=^5x5NZy&8s3SVLK!**A**R3D7#IQ^gA4)550Wha diff --git a/tests/data/small_sky_source/Norder=1/Dir=0/Npix=47.parquet b/tests/data/small_sky_source/Norder=1/Dir=0/Npix=47.parquet index 7fccea01b99eae3473cc1e40dabe9eefaa0c7fbc..7a55e4f099fa3f119a9ab4a6fd93d5daef5cb88b 100644 GIT binary patch delta 465 zcmZ2Jg`;x{N5dAz%=bLp@foR!IR%*&@kW-@?Y=VFPv8EY@fM2k_5~jp`&m$gKq{jY zFY;C^W#*-%R>UXg=alB=6)RaOL@OZ~xP8x8#!swja0`@l6qM33Q*%<{^AdAYfvS-e zPyg_X(RI4~A4Vfa^XWlAit~V!J_u;1Pw)N1XvgH3KK<+;MrB@?bfE|OiSy@J<|C_{uE)X@ zGyM=NlQX07bT&371E8UiKr@}nBg0LzBO`o)Yy&Xg5h&yB=;`Pf5D2u@#nUm+(J{}_ z)zQ(t9OMcE5Z@8R_bms~&Y>V)B8YGSY6MF-IvNHT>+6H013jmI~xU9^1{ z8msVo_mfYKd-gL4j^!QBi)mLRo52ab|v=f}#2JkDN?;9KKQv3;~Wo Fh5#0Qm$v`_ delta 519 zcmeC2!m)e`N5dAz%=i3!@fn!~#mR{!@tJuksTJEx-!n=wVu;TAz}U}%E;`-xD`OPn zt?3iKGK%w{7_#9j<0n>aWFaLT1*Npi)SQ&~yu{p8B`XCb6nm%p{bqEV&ijYah|z4i z1CZi8Bc%@pj?=6EFxoLWIZogGhf!J9(bEwKTtS3mBoI55M~0hbM@INMI{M{LfBlD1 zQ6MqGsVdUQ%^<=kr!>kv%w)U7U&eRrRypO#Mt&J7Zk7c}xj{Lp&Ys1d&R)5R;bFy5 zZkDBy;f6V$S^4RnIhOh8222-VWRjj9%*xclXf*u>E0aOJOCZo{k3eT2S?rkZ=%@{H zk#D(^qoZ@WBTz0G#CJ&t@_`H>-vG#P^a3h$bSefh13&~w!qFvAKTTgB$Z&GZ0EvN2 z199NMAH<3<$|y+o@XLyf2ue!~H)YL^ib!|dK7)-(oLSA5K|-M*v8b>#wL~|$pg=dV us3^Z&p)9qiI5R&_!O%?4K+iz(&h-17Op4RraWb)SGGs_GFa$UT83F)-ZK|&T diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=176.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=176.parquet index 5dd191ea982d98e462da486b95951f18773b4200..cd69e0426fcbbee805960773851b3405e4512466 100644 GIT binary patch delta 471 zcmdmWmvPQr#ti}?JlydasfjrSnHBLymYd~6#2HaVTtn+wP(&unhuBUQh>}*k$Xl(H znU|7U5uco&Q<|GstYoDSt%Pivf7B;dHMrqQItof@nW;G`@p*~4sX)cZY9|-Q2v2TE zNMSUe{4~Lc^MI5-2xzBIR!FpCa!j9`kf_Y-lJ1o5=;-7K1b(iQ7bGh3M1{K*C%fmE zL{xd2ZoZiKj-4kgBgw<5Al1#H(m#0e`AjJkC6fgecR4zk<;#CHVoeanHgb0~;`0)7Q-NAgbxeL3?>2cs zf)S(H#8R6^b=$Aja zE>Te+F~X@T(#Xvq!YHRS%00|v^XkNR?EE?9$wqz|DQ=bpNx4Besgq^WB!qJ;^U)Pg zKAkzA(P(mZmO;HsAkcV^KxZIX?3nK8s135!x7^9m(K+1_D3=W4yQBm8Kn9R+0Ax6N z0Tntr6@!=oAOa-e=n|-(rmqiVI5}p3#6YHjIB?((VnrBb6eN52Wkp5=r6q=&vSvp` rq&sfb&K758w4LmpC&zB4XP{>wd1rEdp33A2c`O`ZQVa|MjzNY1%|Mw5 diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=177.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=177.parquet index b8c7073cdb688ee953cfadec738ee59240dd6d35..3f4b5be311d2051b208671aa3070cdc439ec6375 100644 GIT binary patch delta 441 zcmeyikoCqw)`l&NmS#NM@foR!IR%*&@kW-@d2JZ&r?;Ci-a--HUSQ7H$ATgRQW>Rq zk+)hYGcP5zB0f1kr!+UOSjkEuS_#R(?cFwvpIFu47AWZ`D5Yhl=A^{uCFZ6ARU<2& zzTc72b^1FeMk7Y^>59&bMw|zv^g%#7eR_;DqaBlD`t&)@jLN(&=}zg6j!uq1;O9F1 zjx(boPgJ;Dak6`kNko;W>2_Wh#&_&IVHrstP6erM7M1?N)9XDL#rbnA^O4m}|LesV zGku~DqcfxN^s7FM20%k2fo3|DM~0hbM@IMp*#=;~BT&ZM(bLf}AP{J)i>G6tqhp?< ztD~cPImi_TAig7r?^_O}okKyqL=fQu)CiVvbTkYw*4GC~2YOCF;Lj)~3}U;Xx@dc+ hFQYg!D@c#y^o0S8a=?H%5WuL%;UmSs5a1YO2mt#1h$jF5 delta 505 zcmcbykoDU_)`l&NmS+5X@fn!~#mR{!@tJuksTJE@%@`#aF+{V>8T(k!MW+MR+D~`3 zVU%XPH9g*jQJe?Gqj*Reibo9%g zzRQ_WQ6MqGsVdUQ%^<=kr!>kv%w+p>XU2Ez0y*W$Mt&J7Zk7c}xj{Lp&eI=yGfH77 zo_^n*QF^+P4`U0X(e&j$j0W{Cfk10K0-b?mv17WUqc+HP-*P8MN9S}$pj~W@foR!IR%*&@kW-@KZY>cPtW#eyoDmXJtTm!j|D{tq%unJ zB5$=)W?o8aMSOC8PHAplv67WSv=Wkm+w(&iKe4L8El|=?P)f^8%}I&ROUz9Lszz2k zePTGH@bm@Ij46!f(^+E}jW`cT>4SiF`gDgFMmr|Q^yv*TjLN(&=}zg6j!uq1;O9F1 zKn$ZIPgJ;Dak6`kNko;W>GqE?jPKZa!ZMOPoC;FiEGqqjr@v2OltNK7{b4d=%=FS! zMrTIj={r*y4eA{oBY_4wl}Cn~W=BT&0@(&&z9Y~sZ%0o@$ACbfl`fu+fsT%Oj;@Z5 z?&TmS7=ZYWAii%okai9Q@e)CV3s56i!qL$%z*t`&Bpv7pR_y_jfoKG?oXZ^r9Ua{) z^HKbfo5m>4tOZuUAfZr@SX5Y=TB4g=P@tPwRFq$?P?lO$oSC1eU}&yqpl2X?XZnUr VM#bs-Ga1=98GNJ|7y=xF3<1Hen$Q3M delta 493 zcmdmUnf2CX)`l&Nvi|&h@fn!~#mR{!@tJuksTJEb{TU@0F+}|W82ebzMW+MR+D}&x zWt3LA#apce)*qjopHrHfSFB{E5Uqq}baE);Csu7_+mv(^l+rR&b5i2-5_405dQi1Y zXN+can|>gg(TLG(`ez`;c}7Yf3>>HH#W312IXO7zxBq<&oj0 z*^v>xj*fo$)91x7Dheb9125=QA2j?@nbjsCNkj+TjuC3?z#k(;XePL00>gJ2^T!r#k}W zl0kfzbRZwd0P+oh3`Z}ZLPw`!5HkQofFvAU0`=4M^??j0#|)4d$TScK4*Wr^2&0UG zWDmcr$cUh{#BfvA?5K!z$L-!}jN;6Uw$sxy8Rgi`^bGV2B=1b`&tz1Yz9Ey5g(F>x Kfg!*#$PfSs_L~;~ diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=179.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=179.parquet index 6a730dbea746b0851a62cd3753fa372246b53653..b7e52babf10c9845f75ea565450713b2f8804770 100644 GIT binary patch delta 475 zcmZ3~!`9Nn*06;!HJgV!J|i_Try#Q;-pF#gWg(;e^v&6fw@`$)&&grzV?hxDsf<#* z$Xl(HnU|7U5uco&Q<|GstYoDSt%PLY_U(m?pIFu47AWZ`D5Yhl=A^{uCFZ6ARU<2& zzOjT+WcvRyM#Je+6^urV=F@$F6z2gceGt%2pWacyXvgH3KK(=mqcX2cx>LHNqmv^L z__+i3)csPIk{RiKy~4-ELXQ_>SEuEF;OosUX$OqS8Ouv)I$wD>pGbtT@We zvNST>Fvl}1KixCOG9Ovv^u$KS`HaTX8Jidl>Kz>;f#y4vM~0hbM@IMp*#=;~BhYGZ zM^8t`fIy%NTs$2E9Ub!=T^$|W%Rw$O0P!6`eBW{)?Hmf?C4vYSphmESqoZMfvA#Y? zI?xlW+5;p5(FkTampckNI=Z2{Zu{0IMsa3Vu!8AlTN&kmq4cJeQIEq%ih&`(F~|@A D&_|VM delta 473 zcmZqaVO!M0*06;!HJhI=J|nZBI61K-J~J;RwPJf=HlrjXhUl~$#y%Ew(dj_7_S5qV z8KqTj@m4E=^~WdY=alB=6)RaOL@S{gy}FR`6RQrgZAv-{N@6aTB4eDJ2fp&NVIs?gK$8<+WZJ>pYzU5Aij?U?hK)GZP-z6Q$ z2Qq+s10ciE3#ic1sTjlz01+SwN0&hTG<|&_!^trNBnC1K#DN2U5G%qcqafMCFDo)4 sC@nGElr=jlBHeNOsuM+fT41(l~Io)U5bGrz%j@W0GdLUmH+?% diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=180.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=180.parquet index ae4c4ae4694a84e6830bde79738790a589350e02..3e350eaa0a6b5c3f89d418a4855a6a7fdb55fd14 100644 GIT binary patch delta 454 zcmX?jl4MMzl9fWV5|Uw?!zO%URfC(Yq@$pemYJH95}%itn+jBntah^EMB&ME zry5P(In9XCeDa%VMw|zv^g%#7eX{y=J0{2U$?4OTd0o<-(j6V09D%^kb@KA*iab%_ zZpF#&IVKTRo~D~`On=926qb?X;Z%_7W>M)M>{;yT?3J4s9#$OXW?32;ZkXelm7ngJ zW0{YvZ_MMzl9fWV5~``2y(WBO)j_sLNk>5`Ei*MIB|a}PHx;M_O~+)$Ny3x= zOf{Oke3}uX+2mW(j5yCo>4SmeWYOt%OiqrIL#HdtI(j++fh&k`j09q*^2l)0?8pdT zM@PT>$*t2B1rj5isv?ct3?htjN~7GvOg3+t{*GNBr##unFC)dxvLGopC@0l<@|yWl z7)mFvnk_x~?fe!-qsc`J4C-A1ffjfKIs?gK$8<+WZIHdbY1~LuAfdhXKE5azFAlbt&D>5P| oEiv4bH9IOI-Ep(gLUCqB+sS@QVNl z=B1=o#3$$Hl;-9YD_JQ-D4SiF`ed6Ec1(`xlWR^W^SY!vr8_!0IRb&7>*PHr6nUb;-HMal zb4((tJWV&hJ@Jm6CoCh$!>J(E&7#sjcyjz%asC|3d}LLV@1BpDTzJu$(RlLKiv~dB zB7vqkl}Cn~W=BT&0@(&&z9Ue^+tJg}F(43VpNpqsprd1+qpPE%dpXE{0}$U4#P=-+ z($1kEULuHa0cr$GI64{z80+hUqys%CFSsHm3}U*Wx@L3cC2?j}kPgSmjaTJ>o?mcP OkHbTXfg!*#$PfTAi;`LZ delta 483 zcmdn7n)%vl<_&VY`T62AG7E~66HDSV^HNePHf!ycV8jsg+tba0E;w0hx7}prgVKz* zCfgqr=Rq+f{@^E89b^$D9R;Pd%+#Ee_`Jm2R3$3~B~)uCHy#$A+;u#K(QNYj<3^lk zr1ZhSakAD4J0>T`$yq0qWgR^ofxs0+I7R}oQ+Z^#X?A3UucM=1{^VIF6a^9^oT?&? z+zcX&a!RAz!%Q|GKk<%TAg4Up$S)(s&9WdVHz+67dGf&vQW#1n?>{3wndxE+qtWE{ ziw5;Bfj|p90-b?mv17WUqc+H1-*P8MN9S}$pj9!VoPx}Xcq7Z{;v9_j)2A>o-a--HUdznb&4MBXQW>Rq zk+)hYGcP5zB0f1kr!+UOSjkEuS_#R(?Nd1zKe4L8El|=?P)f^8%}I&ROUz9Lszz2k z{UZ;f@bo*pj7HOc@G%-Onorl}XEfqGAf*oi+Ue8N_!;e(9Mh*S<7ZUnbxC(hcXV`e z1Oh+T>CgBX6?vk<-HMalb4((tJWaQY3oyQ8Hww#0@^C6hb+f4S5B4nfboRf^j~h@$~x=j0V#$^D_$8I~oA(a4L5MTIcQP>F5{` z2(;D3(=pJ|G0)M}(b2sew}~NJ;ACy uKr#@GV3spT&JEQ;+b2shiZio<<)^QfVUz;~%qba0Jq}MP28IB~AVUDD*^6HQ delta 463 zcmaE~lcix7OT!jMHzt0*_>9bg;^f4V_{_YN)QahR9E|qUCo(bKLYLWI&dk`&f-VYD zAEk1Ow^|9TJw7=Ob--iloZaf%tu!}T~LB? zKBLj}jS`Fo^)7)xD?9?7fn>2`x}&2u$Y$SiCr3x;bVs0EGKlYz4&(zFK)wNx;phca z=;%}oVg`T+kc6X4pnjUZK9J$$m;n+4nFiv(fj@{9VU$sj?BSOc84;A07;eg%9Tk!8 ixV=!4QJk64c6zT2qZ}}J*2pmGaimBwFa$UT83F))XN?B{ diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=183.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=183.parquet index ba150b649334ae1cdd9df198281a36268de441d1..d5607913e04251b21678a2ec911587cc7d47eefa 100644 GIT binary patch delta 431 zcmdlzk!9{gmJN*Wc(~&;QWJ9uGArVZEH?|hlVn5{v3TFjf+8|m;GO+s!7tK`7boj} z5$8nc4fyhjRSnKn(os-K%S_EliO);SO;xf|P(re2^825zlQ;h|Vl<30Ca^l7VOhvz*Hv1sxsTEc20lu-%W9QJk3-tYCT?2csOjxt@WZf#jX(6FC?a Ur!U}OWaDJ;lwx2Aa11g801DxY00000 delta 482 zcmbO`k!ANpmJN*W`1#^9G7E~66HDSV^HNePHuJudWW*3Odf&~0E;yO@o&98vFVZTv zc&nAbdgGJxb4qjbij}MsqLt8$b@}p%RR`H3B^?E&w9M3;l=!^F+*F_zG#!%#e~M4; z|EV{5;V&abv&mO}8F8MG(gy>_$^5_Vn4BCZ`~Oy!b@X%u0#^{>7zxBq<&oj0*^v>x zj*fo$lk0yg3M582RYe-P8AKT6lt#ISnQUJ3`yD%fPI&>2V;JEl82YJ=?cEq8KsbWV2!$|ZyNF6lr% zkOAZy02z*6K!uJ@#UN$?hyY1Ax&-Q{>FWa-PL3HMF_38>4jlM{SP@1U1<4+MS&5kiVSQ*8c8EvO~a4^cTo9P+o8A#rlUckYq!Xv}Lz>p%vz!2aVWC#EQ C?w&#b diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=184.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=184.parquet index fb798bce1727af2fa1266bfdd58308bb76a1cf3e..817a8dea1d94f331f44e722a139de6e59c9c3976 100644 GIT binary patch delta 496 zcmeDD%<}pZOT!k%^Kv}g@foR!IR%*&@kW-@7pO4WPnVTvyoDmXolSwUn*~J(q%unJ zB5$=)W?o8aMSOC8PHAplv67WSv=Wkm+vQXlKe4L8El|=?P)f^8%}I&ROUz9Lszz2k zol%ofc=`n`Mx*JC+Kfhw=F_(TDb52@`XHd4KK;EmqaBlD`gDCAMrB@?bfufwRw6BX`Ooa~-s5>e%8x_yBT<2!bqu#6-Rr-D>Bi%S3C>GMn&rBD=2pJ2=w zGhM}$(V5YBdWI>ZLA|46B+x*o^2l)0?8pdTAlm@UcLdty?da*~7!U}w(#6v;(9to^ z(bdt>y&U8O0}$U4#P=-+($1kEULuHa0cr$GI64{z80+hUqys&{sy#q55RG7#bGf6S zqobQ;K8inN%oxR)wZIA(Boqn~iwa9qOLUV93Um{Tit@`9%2JDpGxPHl49)cn^b92L ZOpmu@RGgk?$;igZ;3>tx5a1YO2ms@tm0kb< delta 511 zcmaF;iKX*1OT!k%^K$%r@fn!~#mR{!@tJuksTJFA%P~qaqKj@*VC-f=7n}}MYCrv& z3Zt~jE#7J+u=e=m{G8I*(m0KiyY{QBfc(#PG1oyfW$zifjDsB4`M|aWfUZP_+>>#1f?a0o3ds{MWj1!|7FT3&dg{#UCfeEj@?Yp dK+iz(&U8mhM#bp?mW*ti3@K6!3;~Woh5+~&pNs$i diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=185.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=185.parquet index 5d338aa9dbfd1f6295358208efd82a1765d203f4..ef175cea7a931221dd236f523cffba060e4c57cb 100644 GIT binary patch delta 465 zcmbR6iSyG(&W0AoElfdiJlydasfjrSnHBLymfI8Km?Rldg*xMz`dLtfrUO;kPftr> zl2*LPTdkCtmy%i$pPZjlnwwXwWTg;`0)7Q-P|H z6;Ef(WD=hKCX>l%I!88>5u^EZYaqpWKuRA3w9}`TWi#0^Ii^qFmd&Kh>yqx2?&#>` z2n2qv)4ydiDe^>xyA>z9=a@uPd75rl&tZDUZWNZ0Sj^tAM9D|>Fkx87#>y} z? z5NNB5r(>X_W1gd{qoaE{$Q1@4z9We5TMne1LqWVm5a9yU2$pbkGz>7-*9S=ldV*DZ ufMg&V!7OKxoExfxwl6Ma5@%)w%TM23!6XL^m|GQ0dK|t|3=9E|L52VXDw0(I delta 461 zcmezLk#oW)&W0AoElfdi{Cx2lnFYnki6!xwc`2zC+vDPxBpES8o8y`KSx+P(N#}ZEMc0@ zXf*v`36nv+OCZn+k3eT2S?rkZ=%@{{*|*%u(a|~G5h#}o;=7~+`9KDcZvbRCdI1$W zIu(PM0U!b-;ph^mpQf)5WH>oyfW$zifjDsB4`M|aWfUZP_+>>#1f?a0o3ds{MWj1! gZ!BdJXJ)jWKDUBN4j4RpDwy;*GNc$70vv-30R`BT0RR91 diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=186.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=186.parquet index 9c0cd57e7b55987ea3d66cf38a84e04fddb6d7bc..203db17a1220f72fddcafc732b72f07fea0afb2a 100644 GIT binary patch delta 455 zcmbREgYols#trF-JlydasfjrSnHBLymXo#8Z6|L|yoDmTc}`M23yKI(S%l(6-fE@H zyp+_6_~iVY(%igaB`bw!B_zW(FH8T#ss=Y(Nk>5`Ei*MIB|a}PHx;NDS?%P)4B^Rd zbBrc)<{2@XPqxW3;yfUw4+7fhlgsn$m>kn5Z_iWabxC(hcXV`e1Oh+T$=~x7d7{GI zij&=QOd_g0O*d=gzhgHF%SiHYDoAy+sPqr^EcSHv%1sOpD~@urER75|%<;_1Pxs8R z%tzKYIiP$#qw(YqT4riaN_<{oZYod*#Cb+a9}FBPr{>u)IXO;Vnx`!5=;;Uqt{}oO5{RA3Bg0LzBO`nr z9sTkrU(Hh#NQ`i*iZpUFh%m}2jdBk&+02yxj$I(9JlV)EBgM_KASpK}C)IiKpE4;7 zrIUXZNl&&dZ(%f=e5l-@-X#!dfk&V-kSumgcXZSS+3Q>G(?A?J@CUIXj4}$6J^ZpFBZATr m!%bPUqaxBBH#b#?Gc(#wo>wgg441vtdK_U=3=9E|L52XT;GNL` diff --git a/tests/data/small_sky_source/Norder=2/Dir=0/Npix=187.parquet b/tests/data/small_sky_source/Norder=2/Dir=0/Npix=187.parquet index 53a7fb1208754c23f8392de22cd7e517ef5e06e0..0cff7339c3e4b21c7da165251533a24692863773 100644 GIT binary patch delta 474 zcmdmam1*`>rVTt3c(~&;QWJ9uGArVZEH_I}5NAXcah%x1f+8|mdV=j_{%O*R7kR6d zGV@YWE8>&$b4qjbij}MsqLq+M^PBdGRSj;ql8%B>T4riaN_<{oZYoeQvf9apvxFzj zF`B$_juE5zV63kXk`D9)tM&lNKs16`&gG7Rj*f1Y`6yoSSuW131(s)! zP$)<&DlAPc(M>KW&`m5V$}d+aOD!tS%+FIWG}klGGmyM9xqp@7UbF*!L-_MNLN>*(nS1g;>$F%pQK$|J)~vm+yX9UcAh zC)dtZ6iAG4s){snGl($CDUEUuGugaq?mKp?obqHNzl;<&%Yvlbpqx}^&tgw!uiV7& zu;M5;%hJej!yM17{B+M8%Y1bGlTR(3&uBC`Yneg4OCZojk3eT2S?rkZ=%@{Hh;O-* zqoZ@WBTz0G#CJ&t@_`H>-vG#P^a3h$bSefh13&~w!qFvAKTTgB$Z&GZ0EvN2199NM zAH<3<$|y+o@XLyf2ue!~H)YL^ib!|dthHR6nbCH#+bTJBGd%-61Ias+3s)&lZdk>} O$q*^Uz!2aVWC#E;(5ChP diff --git a/tests/data/small_sky_source/_common_metadata b/tests/data/small_sky_source/_common_metadata index 28747fc8361f0005d16702697de3d11a74b2da84..7ec026d035c0049c18807109c443cd39fad30a7f 100644 GIT binary patch delta 380 zcmbQL{a0(kJs$4(jMT)Og3O9|Bg=_zB^597Rx4%ZrKDEGC+Fvs=H?YESt&#-At~Cd z#rTO;4X#H?M?onqGc_kAJ})sh6{r|l?c|3{!jtc?8BOkGH)1rOe1P4E^MI5-2xzBI z{>^U3e_pS92)xM1{K*C%fmEL{xd2ZeGLjj-4kgBgw<5 zAl1#H(m!~z9IrTkj%7Zws>yTsVJ)g9RnR5^Bi3r9o@@8_8Wlsjv&5oIgoY^1@RI=gbPq3Si;fKFu+(} vA0!>Twi`GB5-<1{neX?qqa@ delta 448 zcmeyXHC21UJs-aKjLd@KT4riaN_<{oZYod&J(bEwKTtS3mBoI55M~0hbM@INM zI{M{Lj^|JmNQ`i*iZpUFh%m}2jdBk&+1$hNj-5ZJJlV)EBgM_KASpK}Cw1~)9tq(b z%Y1ajlh+8$XEd7ZBWO_X5(qTjBhVR07CWXpI%APyY(gIEzp83oB6ep!(bL1~HM vrmWdf5$TSbd4$B78Eq$Ph{~~>=^5x5NZy$oAgaP61B{nWQ3i$p#~?!hZ|{fR diff --git a/tests/data/small_sky_source/_metadata b/tests/data/small_sky_source/_metadata index 8d2e4ef0eae6382f307c9c0e01485616f7e0bf78..b7d9a1706f6de3891356177c680b43d6f071dade 100644 GIT binary patch delta 2629 zcmZ{mYfMvT7{^03OfY0~ zydRcbypUaVi`!%nP29{h&M%s5BUzSxFv~Q$EYZ2>ClkNy{Lgt`EF|;;{GRuDp7Z|R z=k%QP(nIx?ztm^`6xa6+`MrVh;X{44;%&(dmlG;Cs#Gfd>`a*ORJfm%Lf;QdrJ}lD zt-kjZ!zw;gw}gcmZ=3Gf0PPT+e!Nr=_17*o`y7O7}Sqo?PRH1y~uDDfyF5n4Q& zO@tbcM7=UprSIa3$YM~&r{rc05afL+Nw1^*A?pPKw0YHeK^duX1>I(4ta2or`$4nD zXwPltXE7#g=*BJlEQmhY!p~w%KFD^$`n6mWJu3qDnA2yDvZik zLQfIhGxBpF+9~pLAi5~>bHtkrD{q6{1ogRek$!1{0?2B{0YKNy*aGC-iti}s-ZHt< zpoz7`G>CyJwsE?vLv1d7jMdmOpd(caGndw_G18lI&3v{K-V*IMUZnl zz7n9{w!<3Klf~&_7QE!TY}m|aPqVmp2fxBUJK%(x)*Orq^ivMJBoZz>jab90#p01_ z=F;`_ms}0KV}Tow8grEsBZphf*9Wc2WR;$0E20B=$_%Ak9&R=7>9@h0NB`MgUs-xs z^5M>Nf9J!WM;!&qZ1wcVf>pQraRF}iExSm&?9jetKh5IcPK*v0UfTl|P-Yw7^P{#^lesFTQfXeNk!G|HGn#K7VYI6w!02v=U#62R{=L(| zsJM&w-Cm}~?!7GgN4JZSqom|+~|D~k-q#x#=Ju`4-ESQ{e7d}pkFEut)E(*LS9L|*)A%Vok+mu#qZ96*S zIGyf$+v)Uyhd$`o=}X(OKJ=y2i%zu{rG-LmwK|o7wpeQ~`+eu^LPX9?lHY&7Z+HLS zclMmgxhd87wCc4V^4B?wpOpNyRGiu#k-9%=#k-n(P zrh3(&8u6t3lTGu~LmbD|PL5k*1-=to!$(zd9*(apVsgA#;cR?8Pne5u;n<(hs$kZ8 zP5Sk5y7HKWG+pK+rs-rtvsyCJOGy%KPV{iRoH)d>C&@44Nj~C9c5}R)Z06XiQSv1E z^&&pAYM3Y7S!Lnawc5tR-OBUfHKD1@q3QN%Z3sA4~zXtO;@@}q0pMc?Y=o9ezI`j$HoG$kNAU!4Z(KcR9?}Sx}PNIz& zXb~sNjU~`baE5=m*+;n z8((HVTR9GL;;|g8gGX=WD91vM+9l;^Xz4|1!?Ko7OIS{6v6i;yYiL(4I^^<3E>_3o zpaD1L@}2>YFqe_MD7lb_<;YR=&l7*j$I7@I-5j;*Uz@QU9(8ZQa%8!RUR^3R`ZMEk zE8nW8wqofpHME`Gs;&4qa@n;_Yzxt;ZFsAAbmR%q2#C%vm920yc;du%Q4Ayw6-3F| zf~ZZ#3Po*@e6vv02C}PYsbBqDcjH&CZL+9CcJZqGBhhb*m8~S|HY&A9bk?ZUA<@Gn z0w0zr_eP@5QpGTPq!uw^5{vz7Qanr4_oU)Cmwv~Qy*|p8Kgq5#aja%5H1t}TXg$dO za?ur#zm$vCgFILvT0dJ6;=P8*iAr&#K+aT(UV|Lj5hWLPL~Zg?l~RV5Lh{`z(Rz^m z)uQzvKd)Y1($Cdc(v=#CI(EwCkX-uXPTUkm1G}&c!27$f1i)jv1;*8&eHiuDDAP_u zeK0b%2kinT)S^|ua4p&doT`;AlA>h~@h}nH4%KVu_XHQm6UwC5jPE2a=gp$qkbGjV z=r+iNx+r<3PIMfi_v)6cES_nI#5Ra-gFM@SCGb7(H;8US)YmAw{bl3wchX#=-02pH zdYa_!F}%}+O~1nMSqs_#d}zU!4&aGqfpINzr`f2tMaeylon|9rt#YRsCbXeVm>q7D zJI!#aO}1x!OsB0FA~uQMu%b0$I|U2F$gehB81PU#dLg!4`k>v#6JF>*TaYl-fwlmL zI|csUDONbX53NDM?R{ts(6L|a|LOkaXLN>zbBwgso1vq-UgB<6(Jsgkr z2>hqVERUYz!CQ8*&;iFVAN|{5;W*f9rC-(Ua!N2x2nz54rl^RK6j8SpQk(;W&P z8X1iks--?d^uNIi-}KGJ7cTvOXeg8nLw-+i&^bCV;$c%BnL)2Sok4R3N@tam+$O7Uu%RmK8tDvr=s9m@ z`eTx*Gw`O$ME?tv9IxwOTQ_y=W#19vC8PixwH+}Nve!#k%niEE%QOR~OBj%5mPpJI zz;^h80IBOR4jGLM%)|#{uo?()*?{aOU)bGb9d_6|hX#7e;{&~RFS&3rxFJ@xo8Aaz TvyMFo6({Kr#Kg3d&eHz@zTlXb diff --git a/tests/data/small_sky_source/point_map.fits b/tests/data/small_sky_source/point_map.fits index 44dc017133a5673db51f79abac08c6bb62311c78..b7e949c4fc0d99eaa0a9f61bd036a0780115be98 100644 GIT binary patch delta 47649 zcmeHwU5s7VbzblMke9Y=joM^pl)B+KDZ*jVl$U}Hj#{`Hfp9o>f^v*P(xgSBX&p!G zL!*Has6_l=B6dKw2&_p3_%dm2F;ENfp|GYkpz&Mz7CjUw3ltR%6fl9l@XZ9;qC|n9 z{eIS3d*5@(I*$cH0-XDE&e?nIwZ8SOwb$O~+&g#n?;Ky79R0y3md`)`{MOd@zx#>* zu&{J~^89Ds{>^7@j?X{;*{$baIDh^dpIo{A$(0fQFW~h;6b5&kdW|04b`;r}xJpTPf<$A_!qliN>Eda8n_D0u1!Pr2YJ9X^c+PlMv0 zjlAIdNR1Yc-}wF22dCaXT)j7aDNEAy23x>>Ga*TmFa~qudYvD`(=E5=iRkU zoVKrnqs8gryK8G_m;RjpjpB77UWeb0riZVL*QV!w2aQi(8M}w$kG{5gbz%CWAFi%H zO~9TeU{4dUrwQ281ng-7_A~)|nt=V;NWh+EJx{Zqr&-U_tmkRg^EB)E{~_yv3`_t0 zwkrZC4jif)FrI$&%6Q!+0aXK3AvpbIlmynkv%2zJPWAIOO96}1Yx`>#SX9`X^6azA zQ5#qs)ClsztQ{;)@3K~aZcur6d3AjJ(eJJ8e**H{`BYUR#?#9`TtkV1r35zqLiU=C zoOnV(B0HU3fH##bQj z*GFsPQ+dYn@%Hua#{5q>DuZ%aRZ~e*Y_*987-MzGSTKyjX=xeLj7S!=` zACHd4@tSH|p5FP~>e{K2XL{#*YwaVBetvCjwDj#?TV0!~Ls)N9I;sxCGOWA!lymT4UZcT z+g#*0w#o%=`~`mAuyv(QAmawVa|!EIj&5PP@*QX#y1FqvcX4gyRQ3{;FO|^#$;vOKR$+;U&Y**~F8&G6IOZFQkU@ zHKiJU&Mt<>^Lv*v%b^5k(GQsCmGQ=i#-C6@N3xy#;bl`=HkncnKiBtE$mJ=;ZspIy z&Js;>0ZN+4tpeLb4Ex&rJG!pU9+7Y^oxtg1S8IS6Wvj7NX+gH?DqNAhfrHeW|%{4?b{TYp|xj21ZfGx9TjVP1VU z`^r~aVYGCNHZG0cCidtlHSA@|U|N{|ug^tHu^*XwadK3p85>p@cy(h5$ z8IRdHXce(+wB**Z)*cb=)vxAI(SqzP20{BEnJ=W~6UzBH1s#6Q@ALfpyfoeZ;o6!i z{-HyN!vRSp=kv5u+<>bRFG<&kiMV!|Cbwn*&dpC0$v*sxY}Nz{#qnJA!~WL=Ew4RL zEnFkmCfGmxq>TAe%92A%nesJ{=jY*2fg>zja5bZkz-{;^J#GTLNV2;|wkobrs^ZW` z>fcv@w83GxD~(1qUYd5KeBsiGh_Sa$#>jI`1nQ?O6+TYdd2s!v2z0cl$PQXwk)~*; zcac5d1Vok3t*#h}7x;w<9cQpbpU4+{I#b*{9xYFIzqHK|F&{yIgHd_+KFKANccIsZR<#MWOk1Ubgtb2dmF4OO&L;a~FZcx4ySF zF4{<)W*^x|exI)b*JVn*VmoTO!8Dcj|0`5OB&<`o3zfICTn0A``lykP9jq2 z2n=6Y-L4>YN5~f!?Fgs^8Ts}Z0@_Sfs&=GmJ&`e!!sumuPh%+#e<=r7;taTRcswT432(E3L(;#|>Xc_MN}g@?V!scrZKKsqV<>xUohqpRg(m4xs!?VNH(W5z#S%9+& zzB{1F_DEoH6r{E@%S$K773aF{7!Ayf5!KJxf}G$A%Ts9QwgJe2*4SH=G}#vQi4NYc zkhf)?ufr{!w~*!jIo6;~`2xtk^*Ex*Fk8I1x@AwnDb8~o(^BP%!SAWtiorJ*q=)JC z&jT#*g-xf9?;fsxz8vDsD9^5cle(Xm!0w1_@5phA>QMnatjjZE+BLIQhWU@r9 zjvfDS{-%Q9oYY}*b`2l7Dvdr@na_YemmQTz)fEtO_<41uukzzVVRue@3uTgs0iRAm3UdSDr6lt)d)iu1!-f4A8?{`x zZnc~7bLrgKe12{gnHY}nG9|S%7J}ZZt-E;Qw0t+=C zX$>rV+u4&Kr{t)UxlO1v`|o`XZD`UhtRE3Yc=smKdvtD2RjS&JUs;Hbk|>_dL8lVL z3xeaJouUq8T$mnw8F-Z|um$BNRR8UTqGMKP4*$6jGaxZ?0C(xQNKj&hspJG~^lbJz z^zwv6VlyTs8aHPgg_*oC)t%#1dV4`}WZOzAr#qjGXXklsexGgR@k(`@MV6SXzN5*{ z^sR5Mt}xmi$O9&Ynw5VnrF()?lK{<3V-i58 zr68yp;Ka^Voqyz%Z_N0;iM;8e2~tq<**vnoY*tB0(i-(yIow=ztar#99BO@-s(P*o ziNfPhA=r}Z%#l)Fr)h_AO-`5IM zInLC03KPjK*4VQZ%fu*9V8HL}>Gu~3-QFPaWX z!>x{Y+aPpz>>7>2OFH+VF6)1l2sG|GzShK92Z)Tc z2y|^v#3WWwqRe%b2BUaAK9vJkt>2;Xd=9#fMaUU`UwIl0-0FGve~LqQ|HVC}P(mh1 zPz8hl@1q8dDZFYzMczNj7`ivT{BKaF%m#)+1qSK0qveXwH~)Tligj6o#|+;!IRuEe ze3CLKIi6CE2>Ur&CCJAg^Qo94Y?XKKh+9O*9r>LdN}3Lt4d!d)7#QNdXuLAWykJ41 zR{Rk1PwUj2d2JdKQ1_AKD|3d~wZf3ija=VZnBMv3>KZ(Gr@*?qZ4yiLd+z9roQgHg z%)0qM$YEylX_)++jSWBNb*YAY&GYl~hza?$&83t;RgODebdn{Zkhgj1YK~pA(Z$FI z;1^XX-2GBvWQB`WSiGts!oOJztidLzZJTC|6a3g-# z#FX!<@jNL9;5ZBahN}hp^`V#7S~uwbu9e2`3J|fDhwnP5mQ!ST3&nw=?-We{B}FQ8MAIz9d&LRB>Wrd%-47fhtOS zv4=?#78z-`IugN6Ag-!$DY?W;f@|@t)R5Lw2r)r_|8VuuB3WR4bo}+-Slu74#>{3F zk(3ZxwGxsYFR&Ph?e55!r~mOH@<5DXUuqKC8Bb-dwi>j-khAN}@UTfEqkEsFc9b;P z7woi%So=H>5e1$i(o%lsrW5L_dEw0QONqT4dP-YZOm)R;9?$DUGzMwKlDTu@+POYx zf@4!G#uw!p$aW5&y{iz%Gg(ZYuF zkA7WItorP7Ixdfd(e+MUV5-B_a-c!>=DKCEYl<`k(i5q~Rq;aDqt@mz8os0CWn6`- z#JmECE04vRGJu-XLcM%1mhn!1hnfMMDkVl12((6Cku3^N%6tzs!!K4U^`(obAQLb> z%&PG$KVEhr07kxN8i=ehR*DffbRp64LJE+YOXCjv&Chk5ZFuEkhW>B-rJ*(9!cvPJ za*j#^J>fK09d%{xyD_n#Mf^Ui8_JH!6LC;gH(K~?1E-dA$ z3W8JaFL};@%N$vlwqq@tT#M>CFz9U7;1mW6oFlEXqmn@bolo7C*dfV1dA^KO?1cheE27K3xT`L}4UcrPm^dWe(zM4j6|x)BH^ou3pdg9f zk_=;woV?+TjuO<2BK4hmSk40Ix^ct~+U&K=5T>KqOF3!QY%*U(M#^EfQ!fXfD-CP% zx@DMp7le0txf;_35jDFj6w9t@})R5;YBnk%6 z7Z(EL>BUs3u^hH1EN~vSUP?&|*IDTk(lduwA#JfL_Ya=)C^6JsfjTKn0YUiyudFbT zjb~@20lNT!Z@ElhOjz%P%c#XE!wzk!)i?&V;mWa-F?L!RMvgxx!uh#%4o`B5>m-F; zd5*4Nu*aa*9J^*_V~?HUumZDn0vz>d(c)EUupC5Ob()m?=!Ce6v~?6qbeym{_+~?s ze^nFMA3VeJC+aOS*-#YMeyQ=?0>Q!C_{~;LvkdIs`llFp#0EDkE+LktW7`$Bx)h>< zZRt-PBQZt)${$)hZl}~lh^kdm+cpmrh6nh^=65ttI55WG3&V)iUPTCq@z=$WilI8NwfYuUPH!`-%pw-L2u zY^N`v0A-mGar3Hb#q$iv9cBTFTCHmTXE`iqsa7VX_MR!dgMxF046WHuYPap)^A4U= z3a{Q4oG4q|5DeLzxD{^<2iZ+OHFxU%a4J}_qORjC_;q%nh<+#?yp)5^*E~Bvr(sGd z18_J3!75Q8DvCTmydpnmi^HBY9#QAMgsfy!5oO)Ecr2F#V0z#TK(=)78$Od4k*<(5 zwjDT3@>LeQIW8uMgoc5wb1boM%9ezfT1eTsO7T{ya7>>Nx}$EFr`2WR^v75Nc~USO zT-_G0i9qT_Q+R{7xHFHl0N#U0{n) zsj=r+J4)-kIHU-obdh?BSIHK+p5)0Mjgg{lPa-`n(HPEAaslhXDa4tp|GCm8%6i{Q zOjq+}mB(b~QVe0s;cDQMLg|=oSQ#}1octWWv~0|_p@b6AA5%4=54DUc*%7bWQ$jH3 z?*-M=$UsBs5v4)XFPmmxw0!EU3hA;ZBnT?g{tuOyLemXRcqa_(762#SzF47ztFaG% z-@zC&+MABh&DJVD`E1zX?j;UY>sWjBZlZVHB?ItiEF+UWMTchLF<73%{K1*8Qk?7G zTwSm9tkf}37v?p%)LCgqF47Ya^F*G&Rc$NLfkg*Fd3lFI>{fu_GK%C=ESA|F?gI%d z%1N?oZSQ8zCG)6HH4q<{WSMH-=nBI zwn-pW$ST?lQeZQ!PlKpKqaq|)x}}kKj-kv72UNZ9vQF+p><6MbVNnb_jR?@75(A|g zOKZg?jtHq%Uxal@p}KvW?=%tBN&2Hr%)VxB8rGlyALpk38>23-??wuN&qJJ7G#gaKqyo>zN&8hO zeWKQ(&P-^fFXb2mf zYUi=eVhq@4Cvq%-#va~TP-Mke1z%E(lZ$eu#91Nnsq2$DsFvnqWFrqpxrh$*E)IEP zH(@18v0dInr8Y!6{HO?`P<^G~ep7x_dPRF@INHNdbX$e~BY&g8PxanWm~z`O_--x? zBvc|R(NGn*HTg$Rarc7;rM3V90~_Yp{pzab9x)M<7Ui!^M~-2XrS#cE>eXJVbX_V8 zm|{#=LBU-(IKF)yUooQa`BQjg`V{v;KQz2Kw-$Y3jKVHFOO$rlVN~7* zl?nba6sW0BIAn)zeo(&TW=GspkNk{=gX3XFd})Lox<8XTM1F}wfr>zqvBG(pOeVYV zovo!5XiGxk3)NTDg3pCz&;BG`*6|v6OA8i{p{xHR(%48F8pqTVU9vowX#(m=!hKHc zB=T*_-SVB5eRM2xc4*M_ru;r%lMv}TQbOugb@y^Qd(?|&MZsA}*PoAgttBMzWmN5I zk)mZ@v^=pP^o||PWf963&J=%7u2Y`I=U%7lJ1TZ!fOF+)jzrQHPBp=!f~v%p-6{55 zrYD>@=3?CNV85?KAe@7vs1Ta^%ntLj>KDS4Y$q+W)l~6AZFssS*C`i{wJ~;Mnv~w* zSg4)wF?1k{10kvT3n#1UXUo`@8S=6ysH#0`p!3X@&CZ_q zn7;Pa)q~RB&2LysLPB*Q0CS~g7jrM_8DscZMe3`e*bi`rL+cNcn`6xF%^8GI*0*WF zOq)q4TST%Wjg_izca!BW016v!ZW7a?B;Co(FwCEA8@(QH_4MnBY7=44VPIS)dx!?{FRUwg+|GdNG3 z-NiOB#AFH-MvjCo!tkN^J|xHIy$p7FXY-Z+NnKX7)u@qCboaRs(mC|JYqZ8BYK$3% ztQ(7v;tq;)HkFd@n%tJJc|3@Kimgo(!^*3)@j3-Q@_>_6ntTo+ME@;$L$IGTBWvcn z=1&p%cpxz?2TO??W8adG6b?7P8=sM`IE71AOuI&YuBIccWYf&@Ul~s&f<({A#9%n% z+!1q>6e4vya;OvuAiNH&6a}gn2u<1M`6`1dBI8fvd}sL(8%`=S%Bw`c^a&}8cAYch z5cBofNvke|r+D|dXtp*UZfob4cB>zLhh4Zbgy?fa!z*yfVZ4wV{-G`Vx(UI0^7?L~ zzy>e3Ok|a4S)QZ6Yp@gdp4XQbhz}a4r!`|Td>N&{ZnhIP@1&L!QEQ~dUe{=dRCKuH z$S7ZVku?e2hCm?-_(SIi>d1H$;euThE5s?VZd9V9>c^$GgV&u`&s-4c1KYIy!`$V% zp88X+imN48;YG_H*jj|M>eh!IJ}c87e+u-W`yNq})D7lWWT?$b52Du4a?wq<2f*90 zyW~lxNmx377#qI%IgAm0%oW*j+=&NM$Q$iyJg;_9gBOqiUZlO4gI@n$w-(gV(qp2` z-B-r%C=(|g?aN(~LKHOy3eUL=`5%{h=iDrYHQ;UG(#2cg<({_99vQ4*uhfJ6bufM7 zJmH-d>7=F&D&vZ1D)c}tma`0)|$#d_}W* z@zP|7kuH5mt3~jj6`)4B5u(|l2TpNuqP~SOC;T=th>vJnIS1bPySZe(*x)hIzO|~0 zgIk0cbkY`lO)vwOt32;7Q=&cod2&4>*72Y?>CEu<1}Hsf1VwfoBga%f%H{2ysTKe$ zM8)aXYTk4If2(__N${d5PR&SW9YgK1z@d()waOK*MVq*;IitRl4VtuALtK)=OGe$A z6*7i8$ZGgm|j ze&?7UGb)meT}}%G6cfU$HfX}Z7eKY>(0x`THGEA6%T#w?S>s0!oq8DXhV$+bNYKj1#KG94(DQyZstt_ys?S`Hx6pLnfs8M+i zBB+!R2^wkPsNBSbmxgSuR>pimg3%b&OG30>#r7B%?NI*T0*3-wPtQ5@P_leN;$?m! z@vha(az&GgYotP(?`(EGH{e8XqIq`N%6W8Dy44xjV_PaNxn&+b<^rp>%EuKt35*eU zs=nVv$e5JZ9?)%%Nar0JOGRYPGGU6~Efjflw*F{Xq9yHe-3lN!&0y%(V96OSkt%c0 za~KhfEHKCaqA>(4+as%qHZ5$c9V=H^;F$#MI5mLC*7`BT**B`mkhi6XLPeS)O=CFJ za^F0k$!j&;dQGaT3ERqdY3aQd`AnEeos!7=Gk`g`?J-5%R)W6diw1GD61oyLg)E`V zNwZ;Do{m%TI3iEe*Xjqlf!{VNU|4N%Fh_q$QBX$MeAQFk@qJpvNLkB0yxz=fS zJg22`3c2bF)ngJtDkWYsaZaWzbqqO0gJ~&e=%4BA}M!9(^JSuaY zTDD0^Q=*|Hc|4)2J{8KYFsTR80v<1SJr9C;qE@41W1d-K?!%6v+5j=GS>^;ayE zkyhr5uSwN5eMdHoFg0~fO{bhGx6pKkBo{S@R!9`1N8OQKb8-~*wm2k~BB&%<>$+CP z@7}X(s(B1g*U@y8Z7Nc=rTC0FypzrIP8SgIjT5}y@t28mtJJAl9&}46R!W-#3$kVN z!`v*;hL+u*DPH)-cN+Dq%j?WfI%shFrD(AP}?xMK(kYKuPEl6>rpj@8U| z@^1wJBkyf)%7i$sKvg19{wzG5r!26XsO4{^}A;g#ZEzJe}~GbZ2(u?e&h}N|BMHP zSc8M{htmq^`W|GM@US;VFm~RvF98Qog`c}>On8B!oAnFHhO^Js4V<25%uP~BLzHs4kK>YPtuungf*~AIf$OXeXqYBzvcbe`Q0|?g?1 zoM4tbK8o@=B$O#5>6L^~Mg%RcI;QQMmm-aPCk8V+zaWb+RodV~z2FqODOlFu5JOd_ zvi%1ZO(n%Q_}&0(^0`6;Cvse-0I4FVOUX7wBtPM{U~vro$O*b&R_BdX$u%1JSZ7Tl zOT$WwNhZZaW!TK@Y@iH_>PLSzyS~{1cm5ywmKc(Gd)x%c8g*EVG=$9FBUH+9Tl;ch zx*L;xI1EhTwJwo?-*DsuS^B;T)4Ew5pTF>(P&?&88oY0QFgp8SU?44FA@P#!(@0?o{ zd&jT@7>x~yRAq`hR%5GDt{H(l3t{?zw4bGLWg@BVQ;jGA5%8RrG zYYOxpQTmuGyxvW3jS{J(bDes$H-}ey6?{*wfmf^ZrO5L0gszFMJT-l*#@a53*`6G8 zqR|zhD^t`8gDQ}3Glto*K(36MG?lO2q4Igyac=34pb7}~x3 zQ&pYNb;_7qjEUt3@^H-%Q;aCJ&?X=~c)YN=U7?|62RDCy#x?N#1&Dc@5(>J=;aKa# z$Ji4fwBaKWpqZ||knt7XT9>WLqU(EI;zF^4>>4aAtjIOOkEoiuO|SWTJ}++$;gkkK z5ej9N!WWZ-XK+vSS0K0s=b31xOk+W{JXGq7m4cX2C%iZF5)CS3S1bfOwAgJRLY&k+ zk{2E)AAx}$_U@ny*#t}_&4|s^p$_pQnf>8OO5Sr74sR8eZG4`Gwc9^$ogc7Jn2irl zBjP8B58_*mLgO)!SdO(K+wwOPzrEV;Q!Vu%l53jY>{_4Ycjgsd=&Xm%ZwHX-uvT;1 z`hkd(sQ247-d&rJv7gb{d`|Q)hvfVDx{;7sPEjB*V4<`FwdR6ad&K|Y=_p1~Pp&7B zhyygRg42`)K9yThS@Bm@)3^^=x(1vXH<&|pf0*-+To-8SQ{J-CG9ozuA3P|u*b5)Zo* zw58uhB+?Qe&-TzpM%uk)136zK_CE~l6E4+FgICLGGB~#7np9Z6=2b3qDOWhADt?-M zPWUJE14Fb~(+!tON4f{#j5wq_w^q3B#w`ogN~3gBff6bzplS{&SH1GYb5@0(suwwo z%9WLnDEdm9y*sess)WTbL<0EJ@oa%RSP7RMDh3YuGvi60BrNbo4Qn8tAg*4^p43HD zbW1Hqr|4R~&qiwNqJi)GfRpcfu0T}%z{6MU5kqJ?g98T@5+-|rOP<(6 zLh?R4v}mFs)s?ReeJv$&*Y+}Nu3XWXp{y|vGD2Ju+fcD^-THT(=6Fk9$ z9yP2(>rwe?+P((iG%1@heKQBLLW&?City1pS?AKXP|?K>di@mSuGW`Z!j79U*i$kI z&D_o5@q;_wh+E&eW>WV`r(mf?v6dOEk{NG$|8&x0zUJ8S^SJUUO>mt{{G(w9`FUu@ z{61e(pyB82Ex*st0evM?{jMf}U#V=Ho8^@;)O3t%jaQzlMhC7KwY}siW%aS{2eRDJ zygnglKNTcWLuX{5n-D@JB z0y>Zt+5=q1Oa!_(QvG)EX!xDF5mGAwQ6tK*Vi^G0J~_X;l*y=3?&-6MH(tbUZXVpKw-dJ5E0?-bJz-QS{$%<{Xbfjm^qR`a_)<{gnL8E z>_&%xSf$s@IVFyX=P?!|KbK>5b4j2^4M=_IS^Bje>`K3wN)v9t!b{jQg5bhZkx*%?+eS;pq?Uyr_V%LM8tMg+6 zW&?Mqq{UGlVqncYZPrnB0J$QbMJsT0vsn1bWHvcT&Ktg(lM5{k>@p0Eo<`Czi4l&< zPercFs#VI`p0ZV?hvtfEQQ$jQ?za^`@`l9VX7?+Fj7{PY`B9ikiP5a7$a+~3YZ@=v z+K^zbe_x4cEaV3Qse0~^~a-^T&bsM9{_xKfl!m)vX!!;;pzk4ao zHk2?W9e$tZ=kcKYY*q43Oz#2=)VULthrE&c^X1-G%PFH?Nu?Q|*`WQy-x1hw0NkU5 zrlq(*e12??5>GLo^qN^l9?#FQW|d#(YmPQw^ZQ6s^G@7|MTw7CdIgTOmB;C6XG^eH z{BUf>wiplXTPQ#JV(BXsOON8ceYicBXM#z0xpf09cUz8Bm)`%|))aVYN2<jplp!A1aQsH{UOyX)=`YbwyG{b&P`AaWj<-0nP=)=D85{a^3@;Gf{UT3{V!3p#kN6YCDOqFq%*| zU46Yu{5g=$`cz#qvXd8!9kVV|Fen_X+U`*gS`gh!5|MSjvvn#sY@MpN5+&UeP` z$Q!Sfo9g0dm+Gp;kZy_q#WDmk7T^({X*9)}$L?ZbNAh=SggKReUF3e(sdNQ7H*#s@ z?#pHF9mjlVD5B;c<9E=I?7Q$>{R4c*psSN39g?Y~C$u`L2zcv{%3}9l$kAUhIMZHX zr9_lauvf4b1>;pNcb?)XZt`I(Q%4f9e@O{JJ=BOSQc5 zR6qQiPSEk&W75gESwv)*jOYx25phc}Db+;)5PlNh21%umIjEG`peZ-m7Nt4k2*;q`n9`!q!Rxpw68ly${Rk3->6uyTB<8iofxJQK~Z>T6;?iHOW z+HBilg15r%Eg+B^O#ZsH@LRsyXMXW2t|!LIY0^p3-1)i0PG@pyOxl+Kk#F#QGf{4D z7sChj#@Q727?h8|EmH?wZ`cA-v679PmFqyu_wW8vhjD=8w)Ee6rmv`;sq}D@WrM>7 zTYiwp?He?(6X-{q`Sr!55+?QB#i&^)kL^%UgxoM{1PMqvQ3rZEafH4+sr|nAyp~0Y z%@^cM(4Z-^a;35sYX{0H5B>8)-^Zd;+*~Xg(xo$h=1o5oV(znllc%J2qQsBQXUIaj z|9h{z?MOe><7gdsClU%ZhU_M$-RWeeAy395Np0$wNd#<1)9s8jjbyDhy47afrii$; zr*gBWOKSQumjW6b9MV{bTZv+48@eE>izF|?-Zhba(PoY+U(=sJMU}vn6V+5%L)ju| zca+yga6FSK#4V)|2X+B;?>xe^BPGS$u+5PqGW?{&VoRB7=3)I<4C%IwREZu_w78R$ zwv*17945vz67oq=a3V7)xS#t<97vxImL-CEYT2C9Dpk+>fTFrm!cZf-4&{*lc0eC* z#Q;|3Fk$p?a~DwT>;xN2tZ-m78}m$R1kYxPpO1}mg1lrl#qezk(tf52-k2hqCF1EW zek4eayE27!PXO7~W8wzltq3IWE(8Ozo$UzTS#$}Vg|m((kNzZnJbkW|oXU@#$?9g%)A~@Er4{?|z00-$-hRso>t=pfGYUXG_iOf`i&f3t{e|hBvUC}!D z;DgmmN;V9n$25gtaG4i1^aoih#Y~WUM2+67{H*@UrPGB|LG{@&hXXAU680@!1t{>4 zEc^}*Ee5{0^|-92#wP0WQx_kD1Pse7?R9<_UK6rLOR6affG~jfcU4-u%fZF-JplR% z#jzqxfmZfU+@!+?U0(JxhB&}GaW}I%749SrKQ{exB}qu;O=SRy^WbhQGl^5GGE!?a zHK{?wCP#c1)eVSX+O92L?Q*tCSF``HcwqMoKr>E~)4Wm8M4HS>qam{roWkeBpJQ?^ zt3usc8@PIk0y^VI3kZpqge3cnbW!x!#{Wb|6!xtzQkKPU+G6PUk9BJxJC#fJ8kZ{y z@6wQ*nu5e_|LFGZnkjuw5oltfeVKX1%ByNySqL*T)fS8{v2Z0# zLW)i8_RQK+a=|54nCc0jX-vSNpdDJ!M~E#Q9TF*fzAh>akj|t;jYU;IryEmXLk=7M6cJ6QxdQ4H z1T5*Yzg$iLj~)FF+eT`EZJn5QlPC<^+9m!_CHD1n+Kwu_^L0RwhgKDLr0Ih);r?eT z?>dObbvNRc)-+D^6LC~}#sRf6p1Z3!b(AjD>OKr$k~nxk+v!E)!A^s#3Sff8NnQoW1Z zTuegCoIpOrO|wijlm8+uO0p}G7O&8o zaM58OzAYGGzCYHRTjVr4NirkUS$mtl5#jG9g zX$i(LzHrrNVi-&)4q8u&3PL1BrNA*?#OYzUlqU?sU_-Vz(#{)X!xBa?!n+O4{W(4( z84YVrPD`IJQ0096C%kGC@vB+fIK0|6mKh&XV|BP?7o{_o!-UMtsLD1m=Sn=bJ(SJ3 zMcH}MnA0CEcM_FAX|pXsevI$-F$H?lrTR!gr0N;5^MLOGd3e_P zgi>1pr5G%9sI*GS(t7#4H<}$4b+uT!wl3GBD%)mIg9G>WRre9L}!& zu-t|3H8a95vOB7^rHU4}ZnlUgg=Bxh>S@w?EpHdn#7DJJF=T{105TS;vRg{--9|6c z{~swXx_fR~cxC*0tKRHs6h|%^iMhRE^iP20vaaPg%lzr&E_R79n4%2I=@w^&dP7`@ZB@vJf>o<(=n376x*PkL@(T1WC}aEr zBb8(r#}3Ct-5yo3B^A$QLNSS%r>PC9awB#GAg(DcJUI2IH7jg9!hijQCSJ?-u=~j` Gp8CIV$bBCG delta 4282 zcmZWsO>A4o5#~JdNnHKDC)<2_mSa->d#0S4l|l`T1t}VCzQp!GkOJf6?QYN~4owt$RI>yj*E?Y~uEh`hT9Mwej(4?ajVm zwJ!i{9=0sD95x?Y2e!_wz99eBtX;md>Q~`(``VHF{+9RVdSP|F0N6ZiS!_9MKDG{Q zo!Ii&47M(8-Pn2>>xJGwvzYtv_1i0j1|-I+E`IoT4pw%~sow{bFmqFb!rzC=61}gKheE{VTWE z3lnzg@?HOdc}BKFU&PocXS@*=wlJ zTBzb=Ib)D%3uf$%f2{gZ_+?#|u@{AW>cIgVmUD0hX(Om&zX9F+y&pFk5Wl*IEidz7 zyNPdILM=S*3@^#zl#GW^LIN*fUt}EzFo4n`Xp`X=0KSQ{4bR%(Z+^+z6M+{%5s4#{ zo@O`S`OXcb3hd@RCJ}aJ+@i2T&%a_VQ1E{~ft~-24!iyLxBS3`35u&i_?!q(w=1^SufFXk zeb7g2Fo6A5td1bG88zMk4s7T3`S6HrYb?#Ok4Fr27hr_tox;9tE(j%EG=U+zdGH;- z##quYYTMU8z;LYtbhTCr){E6->iE8hASYWwc=-KiehDLEB7KlzbUB5G1-q8L=?CQ( z^%*ku$*6EODf2TZy67>}psCB^VZ3t9C<-6J+5c>H5E*J7bn(0A#TZW`Nu71@iqVtE zU!6%F19Uqzr4UUIZlDw#cCuz(FNrJ(ht!(*035)jMtT)O(1^|aV-E<_BADZz9T^U_ zRtgUdJc~8`>=gu4y%C0V9-S@w@Q3*X7|g*KoUHzIh|UO~8v ztbOGZzcj2y-HRPx6JeARr?Sq?3E#qLJ^XV7+Xe{KA~=fu1`P7|nKl9GLkRVj%6J^N z=fZ+cWvsLitUO6uM{u&?iJ<3c;=@T~tMCa4JjLHbxN$uTl$I-u*MqkIKflCGGRxb9 z&=z7%sA^&jOw#+R7fNa@y6TcX$ z?UnH$y@mri0(TWSi%XF!WKyrM0-0f%eJpSLf4zZ~=4p{Zh>>X2|Kg%{T$>cHmf>_ORPPkOLk-dWTZ_%!y;sSnr z0GIIe3Q*c!11FA)xd_OcFuG3Qe#9G(fe>WZ=v`>%BM7jsTdK_#;`ik|%`Z>MoPsiv zlA0nW0#<2?MbEMC7x~~igT%bVR1Z=&(tJC4506<4^*LJCkTK@OFqiLP_%yi&UG3&; zm*8{@(Fy(!OD=J$oP*RLa@$NV($}TH$1Jh&nsIDBE&Z%~Pm?n4KJ%#e{ zm_X+;(zPobB!Bdr=8aaqq_89{aVeG5!RTKJmta`spT+9r8hKiG5H>bU_*FSeFL)G} zJ9BLMYFSUyd+7z$e7a5!y5tNJ?nMOpPca7&Qnp{}aUKpAkU^1<{;U;;)n`;>Bp-?j z8JFcz+tc9cs8EY6t=#zLRgxCcqwGRYfXcNf);}l8t6@V9>K4M>5h!v9+Zj|O3JN?&N`0jC} zja*3u1F}Z7>jsy4QLL4v^66O)%IM!+9>}w|1vVVw33zaA!`Kix1d&8lLx!Sc7u){r u+X#_boYLRY1Pt5pL!cPE?h2+|h}AQ-YMhZvjS!FB{^ukA4RoDbv;PG!v>|r@ diff --git a/tests/data/small_sky_source/provenance_info.json b/tests/data/small_sky_source/provenance_info.json index aa93b408..5998439c 100644 --- a/tests/data/small_sky_source/provenance_info.json +++ b/tests/data/small_sky_source/provenance_info.json @@ -5,37 +5,37 @@ "epoch": "J2000", "ra_column": "source_ra", "dec_column": "source_dec", - "version": "0.3.4.dev17+g922a4b7.d20240529", - "generation_date": "2024.05.29", + "version": "0.3.10.dev6+g5cb658f", + "generation_date": "2024.09.20", "tool_args": { - "tool_name": "hipscat_import", - "version": "0.3.3.dev10+gd573bcd", + "tool_name": "hats_import", + "version": "0.3.6.dev20+gc573604", "runtime_args": { "catalog_name": "small_sky_source", "output_path": ".", "output_artifact_name": "small_sky_source", - "tmp_dir": "/tmp/tmpgasth6x_", - "dask_tmp": "", + "tmp_dir": "/tmp/tmp6vxk1tuz", + "dask_tmp": null, "dask_n_workers": 1, "dask_threads_per_worker": 1, - "catalog_path": "./small_sky_source", - "tmp_path": "/tmp/tmpgasth6x_/small_sky_source/intermediate", + "catalog_path": "small_sky_source", + "tmp_path": "/tmp/tmp6vxk1tuz/small_sky_source/intermediate", "epoch": "J2000", "catalog_type": "source", - "input_path": "../../../hipscat-import/tests/hipscat_import/data/small_sky_source", + "input_path": "../../../hipscat-import/tests/data/small_sky_source", "input_paths": [ - "file:///home/delucchi/git/fits/tests/data/../../../hipscat-import/tests/hipscat_import/data/small_sky_source/small_sky_source.csv" + "../../../hipscat-import/tests/data/small_sky_source/small_sky_source.csv" ], "input_file_list": [], "ra_column": "source_ra", "dec_column": "source_dec", - "use_hipscat_index": false, + "use_healpix_29": false, "sort_columns": null, "constant_healpix_order": -1, "lowest_healpix_order": 0, - "highest_healpix_order": 7, + "highest_healpix_order": 10, "pixel_threshold": 3000, - "mapping_healpix_order": 7, + "mapping_healpix_order": 10, "debug_stats_only": false, "file_reader_info": { "input_reader_type": "CsvReader", @@ -45,6 +45,7 @@ "column_names": null, "type_map": null, "parquet_kwargs": null, + "upath_kwargs": null, "kwargs": {} } } diff --git a/tests/data/small_sky_source_object_index/_common_metadata b/tests/data/small_sky_source_object_index/_common_metadata index 796247f5024173edc7d23d4aa25e1f31396fdf4b..813e610c7a4eab515c8806ccd88d53a057e62b64 100644 GIT binary patch delta 59 zcmew%_(O1mATy)+WFh8xoGyWm>5h(0j*gp8GPf}^R!$CJH{^A91cGFDx6&wgx2nx` M>~V}h)pt3h0Vcu`{r~^~ delta 61 zcmew%_(O1mATy)cWFh8xl8&B^K;Q}@93z3)sXQ{=G&?fFck@o>HfF}m$qwv>n;Y4q L8G-8Wa!3OJL|GDc diff --git a/tests/data/small_sky_source_object_index/_metadata b/tests/data/small_sky_source_object_index/_metadata index 52bfed083250066629c8d228df9dfa17324aa141..cb97bdb359a755afcb4add33c8ddbfa3c5ec5d50 100644 GIT binary patch delta 116 zcmZ1}x>R&R7Nf}Ien!!Wjjm9}T^|TzvIC9sP7L%y{#Ox>#c`Ok~PS#*d0#cj18I744%_i?>nJ4M!=?Da_Ai^;ch@HwK x!%edzBYZcfv9>WYW=?*~VYpe3Gn!GxOwT~iK+;EsfdP&f=5jGG1ULp60syGf8(9DV diff --git a/tests/data/small_sky_source_object_index/index/part.0.parquet b/tests/data/small_sky_source_object_index/index/part.0.parquet index da8af08c2c5bb981f2d8070aaa045e5ba18574fb..d636e4aaeb674d7123315b3fa8d8221c092c725b 100644 GIT binary patch delta 59 zcmdm~uv1|}5ErBQ5h(0j*gqTxZ9W+D<@CnGvsx51cGFDx6&wgx2nzi M`QjLXs^tWv0Ub6GPyhe` delta 61 zcmdm~uv1|}5ErA_n4*?xvY~m3sgb#{iBYnNg@Hk$rI}fxp@FfbrLn1@d79b8ifqD)CbKeT z>k%|c*Fdizv8b>#wM6U$`{WPIijxm8x)5Qw8dE(Hs*W?|5}_)LnHcXpW-cPata6qH zB2-DT663BFti-s>l#Lj7on#}%UD@o!xa$`?G45*TAi`anr8%vbWX$yp^b90@WEdFW Oh#{1lfg!*#$PfS?Wuqwo delta 659 zcmaDT_EBs?n3A@>L83*fiCIcgN|JH1Npg~@MVf`Fagu>WVxob$ahhe)#PVz+R83}P zBuZI7V-Yc?DKXU(qwEk 0)) -@pytest.mark.skip(reason="HIPSCAT2HATS on-disk file discrepancy") def test_load_catalog_small_sky_source(small_sky_source_dir, small_sky_source_schema): """Instantiate a source catalog with 14 pixels""" cat = read_hats(small_sky_source_dir) diff --git a/tests/hats/inspection/test_visualize_catalog.py b/tests/hats/inspection/test_visualize_catalog.py index 9b34e92c..0cac97ea 100644 --- a/tests/hats/inspection/test_visualize_catalog.py +++ b/tests/hats/inspection/test_visualize_catalog.py @@ -8,6 +8,7 @@ # pylint: disable=no-member +@pytest.mark.timeout(10) def test_generate_projections(small_sky_dir, mocker): """Basic test that map data can be generated""" diff --git a/tests/hats/io/test_parquet_metadata.py b/tests/hats/io/test_parquet_metadata.py index 179f7493..6d8f2747 100644 --- a/tests/hats/io/test_parquet_metadata.py +++ b/tests/hats/io/test_parquet_metadata.py @@ -16,7 +16,6 @@ from hats.pixel_math.healpix_pixel import HealpixPixel -@pytest.mark.skip(reason="HIPSCAT2HATS on-disk file discrepancy") def test_write_parquet_metadata(tmp_path, small_sky_dir, small_sky_schema, check_parquet_schema): """Copy existing catalog and create new metadata files for it""" catalog_base_dir = tmp_path / "catalog" @@ -45,7 +44,6 @@ def test_write_parquet_metadata(tmp_path, small_sky_dir, small_sky_schema, check ) -@pytest.mark.skip(reason="HIPSCAT2HATS on-disk file discrepancy") def test_write_parquet_metadata_order1( tmp_path, small_sky_order1_dir, small_sky_schema, check_parquet_schema ): @@ -73,7 +71,6 @@ def test_write_parquet_metadata_order1( ) -@pytest.mark.skip(reason="HIPSCAT2HATS on-disk file discrepancy") def test_write_parquet_metadata_sorted( tmp_path, small_sky_order1_dir, small_sky_schema, check_parquet_schema ): From 02bdb518e011f02d095c4a4ca780f87566ee01af Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Fri, 20 Sep 2024 09:42:49 -0400 Subject: [PATCH 2/7] Add new pylint config. --- src/.pylintrc | 2 ++ tests/.pylintrc | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/.pylintrc b/src/.pylintrc index 6ef32b67..49fb1f5b 100644 --- a/src/.pylintrc +++ b/src/.pylintrc @@ -280,6 +280,8 @@ ignored-parents= # Maximum number of arguments for function / method. max-args=15 +max-positional-arguments=8 + # Maximum number of attributes for a class (see R0902). max-attributes=25 diff --git a/tests/.pylintrc b/tests/.pylintrc index cf331269..8f01cf46 100644 --- a/tests/.pylintrc +++ b/tests/.pylintrc @@ -280,6 +280,8 @@ ignored-parents= # Maximum number of arguments for function / method. max-args=10 +max-positional-arguments=8 + # Maximum number of attributes for a class (see R0902). max-attributes=20 From fb202048ded56177b24139ad1e54200676aeba2b Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Tue, 24 Sep 2024 10:39:23 -0400 Subject: [PATCH 3/7] catalog_info.json -> properties. --- benchmarks/benchmarks.py | 5 +- docs/guide/directory_scheme.rst | 2 +- pyproject.toml | 2 + src/hats/catalog/__init__.py | 3 +- .../catalog/association_catalog/__init__.py | 1 - .../association_catalog.py | 12 +- .../association_catalog_info.py | 43 ------ src/hats/catalog/catalog.py | 12 +- src/hats/catalog/catalog_info.py | 21 --- src/hats/catalog/dataset/__init__.py | 1 - src/hats/catalog/dataset/base_catalog_info.py | 69 --------- .../catalog/dataset/catalog_info_factory.py | 76 --------- src/hats/catalog/dataset/dataset.py | 27 ++-- src/hats/catalog/dataset/table_properties.py | 146 ++++++++++++++++++ .../healpix_dataset/healpix_dataset.py | 18 +-- src/hats/catalog/index/__init__.py | 1 - src/hats/catalog/index/index_catalog.py | 5 - src/hats/catalog/index/index_catalog_info.py | 29 ---- src/hats/catalog/margin_cache/__init__.py | 1 - .../margin_cache/margin_cache_catalog_info.py | 25 --- .../catalog/margin_cache/margin_catalog.py | 44 +----- src/hats/catalog/source_catalog/__init__.py | 1 - .../source_catalog/source_catalog_info.py | 34 ---- src/hats/inspection/almanac_info.py | 14 +- src/hats/io/__init__.py | 4 +- src/hats/io/file_io/__init__.py | 1 - src/hats/io/file_io/file_io.py | 18 --- src/hats/io/paths.py | 24 --- src/hats/io/validation.py | 13 +- src/hats/io/write_metadata.py | 70 --------- src/hats/loaders/read_hats.py | 8 +- tests/conftest.py | 122 ++------------- .../data/info_only/catalog/catalog_info.json | 8 - tests/data/info_only/catalog/properties | 7 + .../data/info_only/dataset/catalog_info.json | 5 - tests/data/info_only/dataset/properties | 7 + .../info_only/index_catalog/catalog_info.json | 7 - tests/data/info_only/index_catalog/properties | 7 + .../info_only/margin_cache/catalog_info.json | 10 -- tests/data/info_only/margin_cache/properties | 9 ++ tests/data/small_sky/catalog_info.json | 8 - tests/data/small_sky/properties | 7 + tests/data/small_sky/provenance_info.json | 53 ------- tests/data/small_sky_order1/catalog_info.json | 8 - tests/data/small_sky_order1/properties | 7 + .../small_sky_order1/provenance_info.json | 53 ------- .../catalog_info.json | 8 - .../data/small_sky_order1_id_index/properties | 7 + .../provenance_info.json | 31 ---- .../small_sky_order1_margin/catalog_info.json | 10 -- tests/data/small_sky_order1_margin/properties | 9 ++ .../provenance_info.json | 31 ---- tests/data/small_sky_source/catalog_info.json | 8 - tests/data/small_sky_source/properties | 7 + .../small_sky_source/provenance_info.json | 53 ------- .../catalog_info.json | 8 - .../small_sky_source_object_index/properties | 7 + .../provenance_info.json | 31 ---- .../catalog_info.json | 12 -- .../small_sky_to_small_sky_order1/properties | 10 ++ .../test_association_catalog.py | 29 +--- .../test_association_catalog_info.py | 66 -------- .../catalog/dataset/test_base_catalog_info.py | 39 ----- .../dataset/test_catalog_info_factory.py | 109 ------------- tests/hats/catalog/dataset/test_dataset.py | 22 +-- .../catalog/dataset/test_table_properties.py | 27 ++++ .../catalog/index/test_index_catalog_info.py | 71 --------- tests/hats/catalog/loaders/test_read_hats.py | 3 - .../test_margin_cache_catalog_info.py | 68 -------- .../margin_cache/test_margin_catalog.py | 22 +-- .../test_source_catalog_info.py | 71 --------- tests/hats/catalog/test_catalog.py | 17 +- tests/hats/catalog/test_catalog_info.py | 43 ------ tests/hats/inspection/test_almanac_info.py | 4 +- tests/hats/io/file_io/test_file_io.py | 12 -- tests/hats/io/file_io/test_file_pointers.py | 15 +- tests/hats/io/test_validation.py | 18 +-- tests/hats/io/test_write_metadata.py | 120 -------------- 78 files changed, 349 insertions(+), 1687 deletions(-) delete mode 100644 src/hats/catalog/association_catalog/association_catalog_info.py delete mode 100644 src/hats/catalog/catalog_info.py delete mode 100644 src/hats/catalog/dataset/base_catalog_info.py delete mode 100644 src/hats/catalog/dataset/catalog_info_factory.py create mode 100644 src/hats/catalog/dataset/table_properties.py delete mode 100644 src/hats/catalog/index/__init__.py delete mode 100644 src/hats/catalog/index/index_catalog_info.py delete mode 100644 src/hats/catalog/margin_cache/__init__.py delete mode 100644 src/hats/catalog/margin_cache/margin_cache_catalog_info.py delete mode 100644 src/hats/catalog/source_catalog/__init__.py delete mode 100644 src/hats/catalog/source_catalog/source_catalog_info.py delete mode 100644 tests/data/info_only/catalog/catalog_info.json create mode 100644 tests/data/info_only/catalog/properties delete mode 100644 tests/data/info_only/dataset/catalog_info.json create mode 100644 tests/data/info_only/dataset/properties delete mode 100644 tests/data/info_only/index_catalog/catalog_info.json create mode 100644 tests/data/info_only/index_catalog/properties delete mode 100644 tests/data/info_only/margin_cache/catalog_info.json create mode 100644 tests/data/info_only/margin_cache/properties delete mode 100644 tests/data/small_sky/catalog_info.json create mode 100644 tests/data/small_sky/properties delete mode 100644 tests/data/small_sky/provenance_info.json delete mode 100644 tests/data/small_sky_order1/catalog_info.json create mode 100644 tests/data/small_sky_order1/properties delete mode 100644 tests/data/small_sky_order1/provenance_info.json delete mode 100644 tests/data/small_sky_order1_id_index/catalog_info.json create mode 100644 tests/data/small_sky_order1_id_index/properties delete mode 100644 tests/data/small_sky_order1_id_index/provenance_info.json delete mode 100644 tests/data/small_sky_order1_margin/catalog_info.json create mode 100644 tests/data/small_sky_order1_margin/properties delete mode 100644 tests/data/small_sky_order1_margin/provenance_info.json delete mode 100644 tests/data/small_sky_source/catalog_info.json create mode 100644 tests/data/small_sky_source/properties delete mode 100644 tests/data/small_sky_source/provenance_info.json delete mode 100644 tests/data/small_sky_source_object_index/catalog_info.json create mode 100644 tests/data/small_sky_source_object_index/properties delete mode 100644 tests/data/small_sky_source_object_index/provenance_info.json delete mode 100644 tests/data/small_sky_to_small_sky_order1/catalog_info.json create mode 100644 tests/data/small_sky_to_small_sky_order1/properties delete mode 100644 tests/hats/catalog/association_catalog/test_association_catalog_info.py delete mode 100644 tests/hats/catalog/dataset/test_base_catalog_info.py delete mode 100644 tests/hats/catalog/dataset/test_catalog_info_factory.py create mode 100644 tests/hats/catalog/dataset/test_table_properties.py delete mode 100644 tests/hats/catalog/index/test_index_catalog_info.py delete mode 100644 tests/hats/catalog/margin_cache/test_margin_cache_catalog_info.py delete mode 100644 tests/hats/catalog/source_catalog/test_source_catalog_info.py delete mode 100644 tests/hats/catalog/test_catalog_info.py diff --git a/benchmarks/benchmarks.py b/benchmarks/benchmarks.py index 1587187b..90354f74 100644 --- a/benchmarks/benchmarks.py +++ b/benchmarks/benchmarks.py @@ -9,9 +9,8 @@ import hats.pixel_math as hist import hats.pixel_math.healpix_shim as hp -from hats.catalog import Catalog, PartitionInfo +from hats.catalog import Catalog, PartitionInfo, TableProperties from hats.catalog.association_catalog.partition_join_info import PartitionJoinInfo -from hats.catalog.catalog_info import CatalogInfo from hats.io.paths import pixel_catalog_files from hats.pixel_math import HealpixPixel from hats.pixel_tree import PixelAlignment, align_trees @@ -29,7 +28,7 @@ def time_test_alignment_even_sky(): def time_test_cone_filter_multiple_order(): """Create a catalog cone filter where we have multiple orders in the catalog""" - catalog_info = CatalogInfo( + catalog_info = TableProperties( **{ "catalog_name": "test_name", "catalog_type": "object", diff --git a/docs/guide/directory_scheme.rst b/docs/guide/directory_scheme.rst index 0b180cc4..dcc6f762 100644 --- a/docs/guide/directory_scheme.rst +++ b/docs/guide/directory_scheme.rst @@ -22,8 +22,8 @@ structure: __ /path/to/catalogs// |__ _common_metadata |__ _metadata - |__ catalog_info.json |__ partition_info.csv + |__ properties |__ Norder=1/ | |__ Dir=0/ | |__ Npix=0.parquet diff --git a/pyproject.toml b/pyproject.toml index cb0f7447..1a944728 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,12 +22,14 @@ dependencies = [ "astropy", "fsspec>=2023.10.0", # Used for abstract filesystems "healpy", + "jproperties", "matplotlib>=3.3,<3.9", "mocpy", "numba>=0.58", "numpy<3", "pandas", "pyarrow>=14.0.1", + "pydantic", "typing-extensions>=4.3.0", "universal-pathlib", ] diff --git a/src/hats/catalog/__init__.py b/src/hats/catalog/__init__.py index 9de6a40a..d23593a2 100644 --- a/src/hats/catalog/__init__.py +++ b/src/hats/catalog/__init__.py @@ -1,8 +1,9 @@ """Catalog data wrappers""" -from .association_catalog.association_catalog import AssociationCatalog +from .association_catalog import AssociationCatalog from .catalog import Catalog from .catalog_type import CatalogType from .dataset.dataset import Dataset +from .dataset.table_properties import TableProperties from .margin_cache.margin_catalog import MarginCatalog from .partition_info import PartitionInfo diff --git a/src/hats/catalog/association_catalog/__init__.py b/src/hats/catalog/association_catalog/__init__.py index 082bad4c..e415adc3 100644 --- a/src/hats/catalog/association_catalog/__init__.py +++ b/src/hats/catalog/association_catalog/__init__.py @@ -1,3 +1,2 @@ from .association_catalog import AssociationCatalog -from .association_catalog_info import AssociationCatalogInfo from .partition_join_info import PartitionJoinInfo diff --git a/src/hats/catalog/association_catalog/association_catalog.py b/src/hats/catalog/association_catalog/association_catalog.py index bd73920f..7308eab4 100644 --- a/src/hats/catalog/association_catalog/association_catalog.py +++ b/src/hats/catalog/association_catalog/association_catalog.py @@ -6,12 +6,11 @@ import pandas as pd import pyarrow as pa from mocpy import MOC -from typing_extensions import TypeAlias from upath import UPath -from hats.catalog.association_catalog.association_catalog_info import AssociationCatalogInfo from hats.catalog.association_catalog.partition_join_info import PartitionJoinInfo from hats.catalog.catalog_type import CatalogType +from hats.catalog.dataset.table_properties import TableProperties from hats.catalog.healpix_dataset.healpix_dataset import HealpixDataset, PixelInputTypes from hats.io import file_io, paths @@ -24,16 +23,11 @@ class AssociationCatalog(HealpixDataset): Catalog, corresponding to each pair of partitions in each catalog that contain rows to join. """ - # Update CatalogInfoClass, used to check if the catalog_info is the correct type, and - # set the catalog info to the correct type - CatalogInfoClass: TypeAlias = AssociationCatalogInfo - catalog_info: CatalogInfoClass - JoinPixelInputTypes = Union[list, pd.DataFrame, PartitionJoinInfo] def __init__( self, - catalog_info: CatalogInfoClass, + catalog_info: TableProperties, pixels: PixelInputTypes, join_pixels: JoinPixelInputTypes, catalog_path=None, @@ -67,7 +61,7 @@ def _get_partition_join_info_from_pixels( @classmethod def _read_args( cls, catalog_base_dir: str | Path | UPath - ) -> Tuple[CatalogInfoClass, PixelInputTypes, JoinPixelInputTypes]: # type: ignore[override] + ) -> Tuple[TableProperties, PixelInputTypes, JoinPixelInputTypes]: # type: ignore[override] args = super()._read_args(catalog_base_dir) partition_join_info = PartitionJoinInfo.read_from_dir(catalog_base_dir) return args + (partition_join_info,) diff --git a/src/hats/catalog/association_catalog/association_catalog_info.py b/src/hats/catalog/association_catalog/association_catalog_info.py deleted file mode 100644 index 62148933..00000000 --- a/src/hats/catalog/association_catalog/association_catalog_info.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from hats.catalog.catalog_type import CatalogType -from hats.catalog.dataset.base_catalog_info import BaseCatalogInfo - - -@dataclass -class AssociationCatalogInfo(BaseCatalogInfo): - """Catalog Info for a HATS Association Catalog""" - - primary_catalog: str | None = None - """Catalog name for the primary (left) side of association""" - - primary_column: str | None = None - """Column name in the primary (left) side of join""" - - primary_column_association: str | None = None - """Column name in the association table that matches the primary (left) side of join""" - - join_catalog: str | None = None - """Catalog name for the joining (right) side of association""" - - join_column: str | None = None - """Column name in the joining (right) side of join""" - - join_column_association: str | None = None - """Column name in the association table that matches the joining (right) side of join""" - - contains_leaf_files: bool = False - """Whether or not the association catalog contains leaf parquet files""" - - required_fields = BaseCatalogInfo.required_fields + [ - "primary_catalog", - "primary_column", - "join_catalog", - "join_column", - "contains_leaf_files", - ] - - DEFAULT_TYPE = CatalogType.ASSOCIATION - REQUIRED_TYPE = CatalogType.ASSOCIATION diff --git a/src/hats/catalog/catalog.py b/src/hats/catalog/catalog.py index cbb71318..86bfe9f5 100644 --- a/src/hats/catalog/catalog.py +++ b/src/hats/catalog/catalog.py @@ -8,12 +8,11 @@ import numpy as np import pyarrow as pa from mocpy import MOC -from typing_extensions import TypeAlias from upath import UPath import hats.pixel_math.healpix_shim as hp -from hats.catalog.catalog_info import CatalogInfo from hats.catalog.catalog_type import CatalogType +from hats.catalog.dataset.table_properties import TableProperties from hats.catalog.healpix_dataset.healpix_dataset import HealpixDataset, PixelInputTypes from hats.pixel_math import HealpixPixel from hats.pixel_math.box_filter import generate_box_moc, wrap_ra_angles @@ -38,14 +37,9 @@ class Catalog(HealpixDataset): HIPS_CATALOG_TYPES = [CatalogType.OBJECT, CatalogType.SOURCE] - # Update CatalogInfoClass, used to check if the catalog_info is the correct type, and - # set the catalog info to the correct type - CatalogInfoClass: TypeAlias = CatalogInfo - catalog_info: CatalogInfoClass - def __init__( self, - catalog_info: CatalogInfoClass, + catalog_info: TableProperties, pixels: PixelInputTypes, catalog_path: str | Path | UPath | None = None, moc: MOC | None = None, @@ -54,7 +48,7 @@ def __init__( """Initializes a Catalog Args: - catalog_info: CatalogInfo object with catalog metadata + catalog_info: TableProperties object with catalog metadata pixels: Specifies the pixels contained in the catalog. Can be either a list of HealpixPixel, `PartitionInfo object`, or a `PixelTree` object catalog_path: If the catalog is stored on disk, specify the location of the catalog diff --git a/src/hats/catalog/catalog_info.py b/src/hats/catalog/catalog_info.py deleted file mode 100644 index b8ddf455..00000000 --- a/src/hats/catalog/catalog_info.py +++ /dev/null @@ -1,21 +0,0 @@ -from dataclasses import dataclass - -from hats.catalog.catalog_type import CatalogType -from hats.catalog.dataset.base_catalog_info import BaseCatalogInfo - - -@dataclass -class CatalogInfo(BaseCatalogInfo): - """Catalog Info for a HEALPix Hive partitioned Catalog""" - - epoch: str = "J2000" - ra_column: str = "ra" - dec_column: str = "dec" - - required_fields = BaseCatalogInfo.required_fields + [ - "epoch", - "ra_column", - "dec_column", - ] - - DEFAULT_TYPE = CatalogType.OBJECT diff --git a/src/hats/catalog/dataset/__init__.py b/src/hats/catalog/dataset/__init__.py index f4bd7619..13a9804e 100644 --- a/src/hats/catalog/dataset/__init__.py +++ b/src/hats/catalog/dataset/__init__.py @@ -1,2 +1 @@ -from .base_catalog_info import BaseCatalogInfo from .dataset import Dataset diff --git a/src/hats/catalog/dataset/base_catalog_info.py b/src/hats/catalog/dataset/base_catalog_info.py deleted file mode 100644 index e5760feb..00000000 --- a/src/hats/catalog/dataset/base_catalog_info.py +++ /dev/null @@ -1,69 +0,0 @@ -from __future__ import annotations - -import dataclasses -from dataclasses import dataclass -from pathlib import Path - -from typing_extensions import Self -from upath import UPath - -from hats.catalog.catalog_type import CatalogType -from hats.io import file_io - - -@dataclass -class BaseCatalogInfo: - """Container class for catalog metadata""" - - catalog_name: str = "" - catalog_type: CatalogType | None = None - total_rows: int | None = None - - DEFAULT_TYPE = None - """The default catalog type for this catalog info type. To be overridden by subclasses. - If specified, we will use this value when no catalog_type is provided.""" - - REQUIRED_TYPE = None - """The required catalog type for this catalog info type. To be overridden by subclasses. - If specified, the catalog MUST have this type.""" - - required_fields = ["catalog_type"] - - def __post_init__(self): - if not self.catalog_type and self.DEFAULT_TYPE: - self.catalog_type = self.DEFAULT_TYPE - elif self.REQUIRED_TYPE and self.catalog_type != self.REQUIRED_TYPE: - raise ValueError(f"Catalog must have type {self.REQUIRED_TYPE}") - self._check_required_fields() - if self.catalog_type not in CatalogType.all_types(): - raise ValueError(f"Unknown catalog type: {self.catalog_type}") - - def __str__(self): - parameters = dataclasses.asdict(self) - formatted_string = "" - for name, value in parameters.items(): - formatted_string += f" {name} {value}\n" - return formatted_string - - @classmethod - def read_from_metadata_file(cls, catalog_info_file: str | Path | UPath) -> Self: - """Read catalog info from the `catalog_info.json` metadata file - - Args: - catalog_info_file: path pointing to the `catalog_info.json` file - - Returns: - A CatalogInfo object with the data from the `catalog_info.json` file - """ - metadata_keywords = file_io.load_json_file(catalog_info_file) - catalog_info_keywords = {} - for field in dataclasses.fields(cls): - if field.name in metadata_keywords: - catalog_info_keywords[field.name] = metadata_keywords[field.name] - return cls(**catalog_info_keywords) - - def _check_required_fields(self): - fields_dict = dataclasses.asdict(self) - for field_name in self.required_fields: - if field_name not in fields_dict or fields_dict[field_name] is None: - raise ValueError(f"{field_name} is required in the Catalog Info and a value must be provided") diff --git a/src/hats/catalog/dataset/catalog_info_factory.py b/src/hats/catalog/dataset/catalog_info_factory.py deleted file mode 100644 index 4729b90b..00000000 --- a/src/hats/catalog/dataset/catalog_info_factory.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import annotations - -import dataclasses -from pathlib import Path -from typing import Optional - -from upath import UPath - -from hats.catalog.association_catalog.association_catalog_info import AssociationCatalogInfo -from hats.catalog.catalog_info import CatalogInfo -from hats.catalog.catalog_type import CatalogType -from hats.catalog.dataset.base_catalog_info import BaseCatalogInfo -from hats.catalog.index.index_catalog_info import IndexCatalogInfo -from hats.catalog.margin_cache.margin_cache_catalog_info import MarginCacheCatalogInfo -from hats.catalog.source_catalog.source_catalog_info import SourceCatalogInfo -from hats.io import file_io, paths - -CATALOG_TYPE_TO_INFO_CLASS = { - CatalogType.OBJECT: CatalogInfo, - CatalogType.SOURCE: SourceCatalogInfo, - CatalogType.ASSOCIATION: AssociationCatalogInfo, - CatalogType.INDEX: IndexCatalogInfo, - CatalogType.MARGIN: MarginCacheCatalogInfo, -} -"""Map of catalog types to their expected subclass of BaseCatalogInfo.""" - - -def create_catalog_info(keywords: dict, catalog_type: Optional[CatalogType] = None) -> BaseCatalogInfo: - """Generate a typed catalog info object from the type specified explicitly or - using ``catalog_type`` keyword. - - Args: - keywords: dictionary of catalog info keywords (e.g. from reading a - ``catalog_info.json`` file). - catalog_type: explicit request for a specific catalog type. if not - provided, we will look for a key ``catalog_type`` in the keywords. - Returns: - populated BaseCatalogInfo of appropriate type. - """ - - if not catalog_type: - if "catalog_type" not in keywords: - raise ValueError("catalog type is required to create catalog info object") - catalog_type = keywords["catalog_type"] - - if catalog_type not in CatalogType.all_types(): - raise ValueError(f"Unknown catalog type: {catalog_type}") - - if catalog_type not in CATALOG_TYPE_TO_INFO_CLASS: # pragma: no cover - raise NotImplementedError(f"Unhandled catalog type: {catalog_type}") - ci_class = CATALOG_TYPE_TO_INFO_CLASS[catalog_type] - catalog_info_keywords = {} - for field in dataclasses.fields(ci_class): - if field.name in keywords: - catalog_info_keywords[field.name] = keywords[field.name] - return ci_class(**catalog_info_keywords) - - -def from_catalog_dir(catalog_base_dir: str | Path | UPath): - """Generate a typed catalog info object from the type specified in the - catalog info file. - - Args: - catalog_base_dir: a path pointing to the base directory of a catalog, - or may point to a ``catalog_info.json`` file directly. - - Returns: - populated BaseCatalogInfo of appropriate type. - """ - if file_io.is_regular_file(catalog_base_dir): - ## This might be the catalog_info.json file - try anyway - metadata_keywords = file_io.load_json_file(catalog_base_dir) - else: - catalog_info_file = paths.get_catalog_info_pointer(catalog_base_dir) - metadata_keywords = file_io.load_json_file(catalog_info_file) - return create_catalog_info(metadata_keywords) diff --git a/src/hats/catalog/dataset/dataset.py b/src/hats/catalog/dataset/dataset.py index 72bbe736..fc567f60 100644 --- a/src/hats/catalog/dataset/dataset.py +++ b/src/hats/catalog/dataset/dataset.py @@ -6,29 +6,22 @@ from typing_extensions import Self from upath import UPath -from hats.catalog.dataset.base_catalog_info import BaseCatalogInfo -from hats.io import file_io, paths +from hats.catalog.dataset.table_properties import TableProperties +from hats.io import file_io class Dataset: - """A base HATS dataset that contains a catalog_info metadata file + """A base HATS dataset that contains a properties file and the data contained in parquet files""" - CatalogInfoClass = BaseCatalogInfo - - def __init__( - self, catalog_info: CatalogInfoClass, catalog_path: str | Path | UPath | None = None - ) -> None: + def __init__(self, catalog_info: TableProperties, catalog_path: str | Path | UPath | None = None) -> None: """Initializes a Dataset Args: - catalog_info: A catalog_info object with the catalog metadata + catalog_info: A TableProperties object with the catalog metadata catalog_path: If the catalog is stored on disk, specify the location of the catalog Does not load the catalog from this path, only store as metadata """ - if not isinstance(catalog_info, self.CatalogInfoClass): - raise TypeError(f"catalog_info type must be {self.CatalogInfoClass}") - self.catalog_info = catalog_info self.catalog_name = self.catalog_info.catalog_name @@ -53,9 +46,8 @@ def read_hats(cls, catalog_path: str | Path | UPath) -> Self: return cls(*args, **kwargs) @classmethod - def _read_args(cls, catalog_base_dir: str | Path | UPath) -> Tuple[CatalogInfoClass]: - catalog_info_file = paths.get_catalog_info_pointer(catalog_base_dir) - catalog_info = cls.CatalogInfoClass.read_from_metadata_file(catalog_info_file) + def _read_args(cls, catalog_base_dir: str | Path | UPath) -> Tuple[TableProperties]: + catalog_info = TableProperties.read_from_dir(catalog_base_dir) return (catalog_info,) @classmethod @@ -66,6 +58,5 @@ def _read_kwargs(cls, catalog_base_dir: str | Path | UPath) -> dict: def _check_files_exist(cls, catalog_base_dir: str | Path | UPath): if not file_io.does_file_or_directory_exist(catalog_base_dir): raise FileNotFoundError(f"No directory exists at {str(catalog_base_dir)}") - catalog_info_file = paths.get_catalog_info_pointer(catalog_base_dir) - if not file_io.does_file_or_directory_exist(catalog_info_file): - raise FileNotFoundError(f"No catalog info found where expected: {str(catalog_info_file)}") + if not file_io.does_file_or_directory_exist(catalog_base_dir / "properties"): + raise FileNotFoundError(f"No properties file found where expected: {str(catalog_base_dir)}") diff --git a/src/hats/catalog/dataset/table_properties.py b/src/hats/catalog/dataset/table_properties.py new file mode 100644 index 00000000..4949d86b --- /dev/null +++ b/src/hats/catalog/dataset/table_properties.py @@ -0,0 +1,146 @@ +from typing import List, Optional + +from jproperties import Properties +from pydantic import BaseModel, ConfigDict, Field, model_validator +from typing_extensions import Self +from upath import UPath + +from hats.catalog.catalog_type import CatalogType +from hats.io import file_io + +## catalog_name, catalog_type, and total_rows are allowed for ALL types +CATALOG_TYPE_ALLOWED_FIELDS = { + CatalogType.OBJECT: ["ra_column", "dec_column"], + CatalogType.SOURCE: ["primary_catalog", "ra_column", "dec_column"], + CatalogType.ASSOCIATION: [ + "primary_catalog", + "primary_column", + "primary_column_association", + "join_catalog", + "join_column", + "join_column_association", + "contains_leaf_files", + ], + CatalogType.INDEX: ["primary_catalog", "indexing_column", "extra_columns"], + CatalogType.MARGIN: ["primary_catalog", "margin_threshold", "ra_column", "dec_column"], +} + +## catalog_name, catalog_type, and total_rows are required for ALL types +CATALOG_TYPE_REQUIRED_FIELDS = { + CatalogType.OBJECT: ["ra_column", "dec_column"], + CatalogType.SOURCE: ["ra_column", "dec_column"], + CatalogType.ASSOCIATION: [ + "primary_catalog", + "primary_column", + "join_catalog", + "join_column", + "contains_leaf_files", + ], + CatalogType.INDEX: ["primary_catalog", "indexing_column"], + CatalogType.MARGIN: ["primary_catalog", "margin_threshold"], +} + + +class TableProperties(BaseModel): + """Container class for catalog metadata""" + + catalog_name: str = Field(alias="obs_collection") + catalog_type: CatalogType = Field(alias="dataproduct_type") + total_rows: int = Field(alias="hats_nrows") + + ra_column: Optional[str] = Field(default=None, alias="hats_col_j2000_ra") + dec_column: Optional[str] = Field(default=None, alias="hats_col_j2000_dec") + + primary_catalog: Optional[str] = Field(default=None, alias="hats_primary_table_url") + """Reference to object catalog. Relevant for nested, margin, association, and index""" + + margin_threshold: Optional[float] = Field(default=None, alias="hats_margin_threshold") + """Threshold of the pixel boundary, expressed in arcseconds.""" + + primary_column: Optional[str] = Field(default=None, alias="hats_col_assn_primary") + """Column name in the primary (left) side of join""" + + primary_column_association: Optional[str] = Field(default=None, alias="hats_col_assn_primary_assn") + """Column name in the association table that matches the primary (left) side of join""" + + join_catalog: Optional[str] = Field(default=None, alias="hats_assn_join_table_url") + """Catalog name for the joining (right) side of association""" + + join_column: Optional[str] = Field(default=None, alias="hats_col_assn_join") + """Column name in the joining (right) side of join""" + + join_column_association: Optional[str] = Field(default=None, alias="hats_col_assn_join_assn") + """Column name in the association table that matches the joining (right) side of join""" + + contains_leaf_files: Optional[bool] = Field(default=None, alias="hats_assn_leaf_files") + """Whether or not the association catalog contains leaf parquet files""" + + indexing_column: Optional[str] = Field(default=None, alias="hats_index_column") + """Column that we provide an index over""" + + extra_columns: Optional[List[str]] = Field(default=None, alias="hats_index_extra_column") + """Any additional payload columns included in index""" + + ## Allow any extra keyword args to be stored on the properties object. + model_config = ConfigDict(extra="allow", populate_by_name=True, use_enum_values=True) + + @model_validator(mode="after") + def check_allowed_and_required(self) -> Self: + """Check that type-specific fields are appropriate, and required fields are set.""" + explicit_keys = set( + self.model_dump(by_alias=False, exclude_none=True).keys() - self.__pydantic_extra__.keys() + ) + + allowed_keys = set( + CATALOG_TYPE_ALLOWED_FIELDS[self.catalog_type] + ["catalog_name", "catalog_type", "total_rows"] + ) + non_allowed = explicit_keys - allowed_keys + if len(non_allowed) > 0: + raise ValueError(f"Unexpected property for table type {self.catalog_type} ({non_allowed})") + + required_keys = set( + CATALOG_TYPE_REQUIRED_FIELDS[self.catalog_type] + ["catalog_name", "catalog_type", "total_rows"] + ) + missing_required = required_keys - explicit_keys + if len(missing_required) > 0: + raise ValueError( + f"Missing required property for table type {self.catalog_type} ({missing_required})" + ) + return self + + def explicit_dict(self): + """Create a dict, based on fields that have been explicitly set, and are not "extra" keys.""" + explicit = self.model_dump(by_alias=False, exclude_none=True, round_trip=True) + extra_keys = self.__pydantic_extra__.keys() + explicit = {(key, val) for key, val in explicit.items() if key not in extra_keys} + return dict(explicit) + + def __str__(self): + """Friendly string representation based on named fields.""" + parameters = self.explicit_dict() + formatted_string = "" + for name, value in parameters.items(): + formatted_string += f" {name} {value}\n" + return formatted_string + + @classmethod + def read_from_dir(cls, catalog_dir: UPath) -> Self: + """Read field values from a java-style properties file.""" + file_path = file_io.get_upath(catalog_dir) / "properties" + if not file_io.does_file_or_directory_exist(file_path): + raise FileNotFoundError(f"No properties file found where expected: {str(file_path)}") + p = Properties() + with file_path.open("rb") as f: + p.load(f, "utf-8") + return cls(**p.properties) + + def to_properties_file(self, catalog_dir: UPath) -> Self: + """Write fields to a java-style properties file.""" + # pylint: disable=protected-access + parameters = self.model_dump(by_alias=True, exclude_none=True) + properties = Properties() + properties.properties = parameters + properties._key_order = parameters.keys() + file_path = file_io.get_upath(catalog_dir) / "properties" + with file_path.open("wb") as _file: + properties.store(_file, "utf-8", timestamp=False) diff --git a/src/hats/catalog/healpix_dataset/healpix_dataset.py b/src/hats/catalog/healpix_dataset/healpix_dataset.py index 6287f50d..8cb893b4 100644 --- a/src/hats/catalog/healpix_dataset/healpix_dataset.py +++ b/src/hats/catalog/healpix_dataset/healpix_dataset.py @@ -1,6 +1,5 @@ from __future__ import annotations -import dataclasses import warnings from pathlib import Path from typing import List, Tuple, Union @@ -9,11 +8,12 @@ import pandas as pd import pyarrow as pa from mocpy import MOC -from typing_extensions import Self, TypeAlias +from typing_extensions import Self from upath import UPath import hats.pixel_math.healpix_shim as hp -from hats.catalog.dataset import BaseCatalogInfo, Dataset +from hats.catalog.dataset import Dataset +from hats.catalog.dataset.table_properties import TableProperties from hats.catalog.partition_info import PartitionInfo from hats.io import file_io, paths from hats.io.file_io import read_parquet_metadata @@ -35,12 +35,9 @@ class HealpixDataset(Dataset): Norder=/Dir=/Npix=.parquet """ - CatalogInfoClass: TypeAlias = BaseCatalogInfo - catalog_info: CatalogInfoClass - def __init__( self, - catalog_info: CatalogInfoClass, + catalog_info: TableProperties, pixels: PixelInputTypes, catalog_path: str | Path | UPath | None = None, moc: MOC | None = None, @@ -49,7 +46,7 @@ def __init__( """Initializes a Catalog Args: - catalog_info: CatalogInfo object with catalog metadata + catalog_info: TableProperties object with catalog metadata pixels: Specifies the pixels contained in the catalog. Can be either a list of HealpixPixel, `PartitionInfo object`, or a `PixelTree` object catalog_path: If the catalog is stored on disk, specify the location of the catalog @@ -92,7 +89,7 @@ def _get_pixel_tree_from_pixels(pixels: PixelInputTypes) -> PixelTree: raise TypeError("Pixels must be of type PartitionInfo, PixelTree, or List[HealpixPixel]") @classmethod - def _read_args(cls, catalog_base_dir: str | Path | UPath) -> Tuple[CatalogInfoClass, PartitionInfo]: + def _read_args(cls, catalog_base_dir: str | Path | UPath) -> Tuple[TableProperties, PartitionInfo]: args = super()._read_args(catalog_base_dir) partition_info = PartitionInfo.read_from_dir(catalog_base_dir) return args + (partition_info,) @@ -184,7 +181,8 @@ def filter_by_moc(self, moc: MOC) -> Self: to None, as updating would require a scan over the new pixel sizes.""" filtered_tree = filter_by_moc(self.pixel_tree, moc) filtered_moc = self.moc.intersection(moc) if self.moc is not None else None - filtered_catalog_info = dataclasses.replace(self.catalog_info, total_rows=None) + filtered_catalog_info = self.catalog_info.model_copy() + filtered_catalog_info.total_rows = None return self.__class__(filtered_catalog_info, filtered_tree, moc=filtered_moc, schema=self.schema) def align( diff --git a/src/hats/catalog/index/__init__.py b/src/hats/catalog/index/__init__.py deleted file mode 100644 index 1270e266..00000000 --- a/src/hats/catalog/index/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .index_catalog_info import IndexCatalogInfo diff --git a/src/hats/catalog/index/index_catalog.py b/src/hats/catalog/index/index_catalog.py index 7f3088ae..4f6e0918 100644 --- a/src/hats/catalog/index/index_catalog.py +++ b/src/hats/catalog/index/index_catalog.py @@ -3,10 +3,8 @@ import numpy as np import pyarrow.compute as pc import pyarrow.dataset as pds -from typing_extensions import TypeAlias from hats.catalog.dataset import Dataset -from hats.catalog.index import IndexCatalogInfo from hats.io import paths from hats.pixel_math import HealpixPixel from hats.pixel_math.healpix_pixel_function import get_pixel_argsort @@ -18,9 +16,6 @@ class IndexCatalog(Dataset): Note that this is not a true "HATS Catalog", as it is not partitioned spatially. """ - CatalogInfoClass: TypeAlias = IndexCatalogInfo - catalog_info: CatalogInfoClass - def loc_partitions(self, ids) -> List[HealpixPixel]: """Find the set of partitions in the primary catalog for the ids provided. diff --git a/src/hats/catalog/index/index_catalog_info.py b/src/hats/catalog/index/index_catalog_info.py deleted file mode 100644 index 54731094..00000000 --- a/src/hats/catalog/index/index_catalog_info.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Catalog Info for a HATS Index table""" - -from dataclasses import dataclass, field -from typing import List - -from hats.catalog.catalog_type import CatalogType -from hats.catalog.dataset.base_catalog_info import BaseCatalogInfo - - -@dataclass -class IndexCatalogInfo(BaseCatalogInfo): - """Catalog Info for a HATS Index table""" - - primary_catalog: str = None - """Reference to object or source catalog""" - - indexing_column: str = None - """Column that we provide an index over""" - - extra_columns: List[str] = field(default_factory=list) - """Any additional payload columns included in index""" - - required_fields = BaseCatalogInfo.required_fields + [ - "primary_catalog", - "indexing_column", - ] - - DEFAULT_TYPE = CatalogType.INDEX - REQUIRED_TYPE = CatalogType.INDEX diff --git a/src/hats/catalog/margin_cache/__init__.py b/src/hats/catalog/margin_cache/__init__.py deleted file mode 100644 index 9ba73bb2..00000000 --- a/src/hats/catalog/margin_cache/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .margin_cache_catalog_info import MarginCacheCatalogInfo diff --git a/src/hats/catalog/margin_cache/margin_cache_catalog_info.py b/src/hats/catalog/margin_cache/margin_cache_catalog_info.py deleted file mode 100644 index ddcc7c27..00000000 --- a/src/hats/catalog/margin_cache/margin_cache_catalog_info.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Catalog Info for a HATS Margin cache table""" - -from dataclasses import dataclass - -from hats.catalog.catalog_info import CatalogInfo -from hats.catalog.catalog_type import CatalogType - - -@dataclass -class MarginCacheCatalogInfo(CatalogInfo): - """Catalog Info for a HATS Margin Cache table""" - - primary_catalog: str = None - """Reference to object or source catalog""" - - margin_threshold: float = None - """Threshold of the pixel boundary, expressed in arcseconds.""" - - required_fields = CatalogInfo.required_fields + [ - "primary_catalog", - "margin_threshold", - ] - - DEFAULT_TYPE = CatalogType.MARGIN - REQUIRED_TYPE = CatalogType.MARGIN diff --git a/src/hats/catalog/margin_cache/margin_catalog.py b/src/hats/catalog/margin_cache/margin_catalog.py index 6149f97b..65858e47 100644 --- a/src/hats/catalog/margin_cache/margin_catalog.py +++ b/src/hats/catalog/margin_cache/margin_catalog.py @@ -1,16 +1,10 @@ from __future__ import annotations -from pathlib import Path - -import pyarrow as pa from mocpy import MOC -from typing_extensions import Self, TypeAlias -from upath import UPath +from typing_extensions import Self import hats.pixel_math.healpix_shim as hp -from hats.catalog.catalog_type import CatalogType -from hats.catalog.healpix_dataset.healpix_dataset import HealpixDataset, PixelInputTypes -from hats.catalog.margin_cache import MarginCacheCatalogInfo +from hats.catalog.healpix_dataset.healpix_dataset import HealpixDataset from hats.pixel_tree.moc_utils import copy_moc @@ -23,40 +17,6 @@ class MarginCatalog(HealpixDataset): accuracy. """ - # Update CatalogInfoClass, used to check if the catalog_info is the correct type, and - # set the catalog info to the correct type - CatalogInfoClass: TypeAlias = MarginCacheCatalogInfo - catalog_info: CatalogInfoClass - - def __init__( - self, - catalog_info: CatalogInfoClass, - pixels: PixelInputTypes, - catalog_path: str | Path | UPath | None = None, - moc: MOC | None = None, - schema: pa.Schema | None = None, - ) -> None: - """Initializes a Margin Catalog - - Args: - catalog_info: CatalogInfo object with catalog metadata - pixels: Specifies the pixels contained in the catalog. Can be either a - list of HealpixPixel, `PartitionInfo object`, or a `PixelTree` object - catalog_path: If the catalog is stored on disk, specify the location of the catalog - Does not load the catalog from this path, only store as metadata - moc (mocpy.MOC): MOC object representing the coverage of the catalog - schema (pa.Schema): The pyarrow schema for the catalog - """ - if catalog_info.catalog_type != CatalogType.MARGIN: - raise ValueError(f"Catalog info `catalog_type` must equal {CatalogType.MARGIN}") - super().__init__( - catalog_info, - pixels, - catalog_path=catalog_path, - moc=moc, - schema=schema, - ) - def filter_by_moc(self, moc: MOC) -> Self: """Filter the pixels in the margin catalog to only include the margin pixels that overlap with the moc diff --git a/src/hats/catalog/source_catalog/__init__.py b/src/hats/catalog/source_catalog/__init__.py deleted file mode 100644 index 1e1a4ac8..00000000 --- a/src/hats/catalog/source_catalog/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .source_catalog_info import SourceCatalogInfo diff --git a/src/hats/catalog/source_catalog/source_catalog_info.py b/src/hats/catalog/source_catalog/source_catalog_info.py deleted file mode 100644 index 7f7b03c9..00000000 --- a/src/hats/catalog/source_catalog/source_catalog_info.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Catalog Info for a HATS Source (detection/timeseries) table""" - -from dataclasses import dataclass - -from hats.catalog.catalog_info import CatalogInfo -from hats.catalog.catalog_type import CatalogType - - -@dataclass -class SourceCatalogInfo(CatalogInfo): - """Catalog Info for a HATS Source (detection/timeseries) table. - - Includes some optional specification for timeseries-level columns. - """ - - primary_catalog: str = None - """Object catalog reference""" - - mjd_column: str = "" - """Column name for time of observation""" - - band_column: str = "" - """Column name for photometric band""" - - mag_column: str = "" - """Column name for magnitude measurement""" - - mag_err_column: str = "" - """Column name for error in magnitude measurement""" - - DEFAULT_TYPE = CatalogType.SOURCE - REQUIRED_TYPE = CatalogType.SOURCE - - ## NB: No additional required columns. diff --git a/src/hats/inspection/almanac_info.py b/src/hats/inspection/almanac_info.py index a5a227a9..bfc7626f 100644 --- a/src/hats/inspection/almanac_info.py +++ b/src/hats/inspection/almanac_info.py @@ -1,6 +1,5 @@ from __future__ import annotations -import dataclasses import os from dataclasses import dataclass, field from typing import List @@ -8,8 +7,7 @@ import yaml from typing_extensions import Self -from hats.catalog.dataset import catalog_info_factory -from hats.catalog.dataset.base_catalog_info import BaseCatalogInfo +from hats.catalog.dataset.table_properties import TableProperties from hats.io import file_io @@ -40,11 +38,8 @@ class AlmanacInfo: catalog_info: dict = field(default_factory=dict) - catalog_info_object: BaseCatalogInfo | None = None - def __post_init__(self): if len(self.catalog_info): - self.catalog_info_object = catalog_info_factory.create_catalog_info(self.catalog_info) if self.catalog_info and "primary_catalog" in self.catalog_info and not self.primary: self.primary = self.catalog_info["primary_catalog"] if self.catalog_info and "join_catalog" in self.catalog_info and not self.join: @@ -76,15 +71,12 @@ def get_default_dir() -> str: @classmethod def from_catalog_dir(cls, catalog_base_dir: str) -> Self: """Create almanac information from the catalog information found at the target directory""" - catalog_info = catalog_info_factory.from_catalog_dir( - catalog_base_dir=file_io.get_upath(catalog_base_dir) - ) + catalog_info = TableProperties.read_from_dir(catalog_base_dir) args = { "catalog_path": catalog_base_dir, "catalog_name": catalog_info.catalog_name, "catalog_type": catalog_info.catalog_type, - "catalog_info_object": catalog_info, - "catalog_info": dataclasses.asdict(catalog_info), + "catalog_info": catalog_info.explicit_dict(), } return cls(**args) diff --git a/src/hats/io/__init__.py b/src/hats/io/__init__.py index 38720b7c..4d245881 100644 --- a/src/hats/io/__init__.py +++ b/src/hats/io/__init__.py @@ -8,13 +8,11 @@ ) from .paths import ( create_hive_directory_name, - get_catalog_info_pointer, get_common_metadata_pointer, get_parquet_metadata_pointer, get_partition_info_pointer, get_point_map_file_pointer, - get_provenance_pointer, pixel_catalog_file, pixel_directory, ) -from .write_metadata import write_catalog_info, write_partition_info, write_provenance_info +from .write_metadata import write_partition_info diff --git a/src/hats/io/file_io/__init__.py b/src/hats/io/file_io/__init__.py index 095c0fa6..0f742317 100644 --- a/src/hats/io/file_io/__init__.py +++ b/src/hats/io/file_io/__init__.py @@ -2,7 +2,6 @@ delete_file, load_csv_to_pandas, load_csv_to_pandas_generator, - load_json_file, load_text_file, make_directory, read_fits_image, diff --git a/src/hats/io/file_io/file_io.py b/src/hats/io/file_io/file_io.py index e496155b..6335b2ce 100644 --- a/src/hats/io/file_io/file_io.py +++ b/src/hats/io/file_io/file_io.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json import tempfile import warnings from collections.abc import Generator @@ -93,23 +92,6 @@ def load_text_file(file_pointer: str | Path | UPath, encoding: str = "utf-8"): return text_file -def load_json_file(file_pointer: str | Path | UPath, encoding: str = "utf-8") -> dict: - """Load a json file to a dictionary - - Args: - file_pointer: location of file to read - encoding: string encoding method used by the file - Returns: - dictionary of key value pairs loaded from the JSON file - """ - file_pointer = get_upath(file_pointer) - json_dict = None - with file_pointer.open("r", encoding=encoding) as json_file: - json_dict = json.load(json_file) - - return json_dict - - def load_csv_to_pandas(file_pointer: str | Path | UPath, **kwargs) -> pd.DataFrame: """Load a csv file to a pandas dataframe diff --git a/src/hats/io/paths.py b/src/hats/io/paths.py index 35bd3d23..d0a8de7b 100644 --- a/src/hats/io/paths.py +++ b/src/hats/io/paths.py @@ -20,10 +20,8 @@ JOIN_DIR_DIRECTORY_PREFIX = "join_Dir" JOIN_PIXEL_DIRECTORY_PREFIX = "join_Npix" -CATALOG_INFO_FILENAME = "catalog_info.json" PARTITION_INFO_FILENAME = "partition_info.csv" PARTITION_JOIN_INFO_FILENAME = "partition_join_info.csv" -PROVENANCE_INFO_FILENAME = "provenance_info.json" PARQUET_METADATA_FILENAME = "_metadata" PARQUET_COMMON_METADATA_FILENAME = "_common_metadata" POINT_MAP_FILENAME = "point_map.fits" @@ -221,17 +219,6 @@ def create_hive_directory_name(base_dir, partition_token_names, partition_token_ return get_upath(base_dir).joinpath(*partition_tokens) -def get_catalog_info_pointer(catalog_base_dir: str | Path | UPath) -> UPath: - """Get file pointer to `catalog_info.json` metadata file - - Args: - catalog_base_dir: pointer to base catalog directory - Returns: - File Pointer to the catalog's `catalog_info.json` file - """ - return get_upath(catalog_base_dir) / CATALOG_INFO_FILENAME - - def get_partition_info_pointer(catalog_base_dir: str | Path | UPath) -> UPath: """Get file pointer to `partition_info.csv` metadata file @@ -243,17 +230,6 @@ def get_partition_info_pointer(catalog_base_dir: str | Path | UPath) -> UPath: return get_upath(catalog_base_dir) / PARTITION_INFO_FILENAME -def get_provenance_pointer(catalog_base_dir: str | Path | UPath) -> UPath: - """Get file pointer to `provenance_info.json` metadata file - - Args: - catalog_base_dir: pointer to base catalog directory - Returns: - File Pointer to the catalog's `provenance_info.json` file - """ - return get_upath(catalog_base_dir) / PROVENANCE_INFO_FILENAME - - def get_common_metadata_pointer(catalog_base_dir: str | Path | UPath) -> UPath: """Get file pointer to `_common_metadata` parquet metadata file diff --git a/src/hats/io/validation.py b/src/hats/io/validation.py index 985c1713..7e044de5 100644 --- a/src/hats/io/validation.py +++ b/src/hats/io/validation.py @@ -7,7 +7,7 @@ from upath import UPath import hats.pixel_math.healpix_shim as hp -from hats.catalog.dataset.catalog_info_factory import from_catalog_dir +from hats.catalog.dataset.table_properties import TableProperties from hats.catalog.partition_info import PartitionInfo from hats.io import get_common_metadata_pointer, get_parquet_metadata_pointer, get_partition_info_pointer from hats.io.file_io import read_parquet_dataset @@ -37,7 +37,7 @@ def is_valid_catalog( progress, and approximate sky coverage? Returns: - True if both the catalog_info and partition_info files are + True if both the properties and partition_info files are valid, False otherwise """ if not strict: @@ -65,7 +65,7 @@ def handle_error(msg): is_valid = False if not is_catalog_info_valid(pointer): - handle_error("catalog_info.json file does not exist or is invalid.") + handle_error("properties file does not exist or is invalid.") if not is_partition_info_valid(pointer): handle_error("partition_info.csv file does not exist.") @@ -114,6 +114,7 @@ def handle_error(msg): "_common_metadata", "_metadata", "catalog_info.json", + "properties", "provenance_info.json", "partition_info.csv", "point_map.fits", @@ -155,17 +156,17 @@ def handle_error(msg): def is_catalog_info_valid(pointer: str | Path | UPath) -> bool: - """Checks if catalog_info is valid for a given base catalog pointer + """Checks if properties file is valid for a given base catalog pointer Args: pointer (UPath): pointer to base catalog directory Returns: - True if the catalog_info file exists, and it is correctly formatted, + True if the properties file exists, and it is correctly formatted, False otherwise """ try: - from_catalog_dir(pointer) + TableProperties.read_from_dir(pointer) except (FileNotFoundError, ValueError, NotImplementedError): return False return True diff --git a/src/hats/io/write_metadata.py b/src/hats/io/write_metadata.py index 64664669..45cd76ab 100644 --- a/src/hats/io/write_metadata.py +++ b/src/hats/io/write_metadata.py @@ -2,10 +2,6 @@ from __future__ import annotations -import dataclasses -import json -from datetime import datetime -from importlib.metadata import version from pathlib import Path import numpy as np @@ -13,72 +9,6 @@ from upath import UPath from hats.io import file_io, paths -from hats.pixel_math.healpix_pixel import HealpixPixel - - -class HatsEncoder(json.JSONEncoder): - """Special json encoder for types commonly encountered with hats. - - NB: This will only be used by JSON encoding when encountering a type - that is unhandled by the default encoder. - """ - - def default(self, o): - if isinstance(o, Path): - return str(o) - if isinstance(o, (type, np.dtype, pd.core.dtypes.base.ExtensionDtype)): - return str(o) - if isinstance(o, HealpixPixel): - return str(o) - if np.issubdtype(o.dtype, np.integer): - return int(o) - if isinstance(o, np.ndarray): - return o.tolist() - return super().default(o) # pragma: no cover - - -def write_json_file(metadata_dictionary: dict, file_pointer: str | Path | UPath): - """Convert metadata_dictionary to a json string and print to file. - - Args: - metadata_dictionary (:obj:`dictionary`): a dictionary of key-value pairs - file_pointer (str): destination for the json file - """ - dumped_metadata = json.dumps(metadata_dictionary, indent=4, cls=HatsEncoder) - file_io.write_string_to_file(file_pointer, dumped_metadata + "\n") - - -def write_catalog_info(catalog_base_dir, dataset_info): - """Write a catalog_info.json file with catalog metadata - - Args: - catalog_base_dir (str): base directory for catalog, where file will be written - dataset_info (:obj:`BaseCatalogInfo`) base metadata for the catalog - """ - metadata = dataclasses.asdict(dataset_info) - catalog_info_pointer = paths.get_catalog_info_pointer(catalog_base_dir) - - write_json_file(metadata, catalog_info_pointer) - - -def write_provenance_info(catalog_base_dir: str | Path | UPath, dataset_info, tool_args: dict): - """Write a provenance_info.json file with all assorted catalog creation metadata - - Args: - catalog_base_dir (str): base directory for catalog, where file will be written - dataset_info (:obj:`BaseCatalogInfo`) base metadata for the catalog - tool_args (:obj:`dict`): dictionary of additional arguments provided by the tool creating - this catalog. - """ - metadata = dataclasses.asdict(dataset_info) - metadata["version"] = version("hats") - now = datetime.now() - metadata["generation_date"] = now.strftime("%Y.%m.%d") - - metadata["tool_args"] = tool_args - - metadata_pointer = paths.get_provenance_pointer(catalog_base_dir) - write_json_file(metadata, metadata_pointer) def write_partition_info(catalog_base_dir: str | Path | UPath, destination_healpix_pixel_map: dict): diff --git a/src/hats/loaders/read_hats.py b/src/hats/loaders/read_hats.py index 278a0b3a..5353f044 100644 --- a/src/hats/loaders/read_hats.py +++ b/src/hats/loaders/read_hats.py @@ -5,9 +5,8 @@ from upath import UPath -from hats import io from hats.catalog import AssociationCatalog, Catalog, CatalogType, Dataset, MarginCatalog -from hats.catalog.dataset import BaseCatalogInfo +from hats.catalog.dataset.table_properties import TableProperties from hats.catalog.index.index_catalog import IndexCatalog CATALOG_TYPE_TO_CLASS = { @@ -41,10 +40,7 @@ def read_hats(catalog_path: str | Path | UPath, catalog_type: CatalogType | None def _read_dataset_class_from_metadata(catalog_base_path: str) -> CatalogType: - catalog_base_dir = io.file_io.get_upath(catalog_base_path) - catalog_info_path = io.paths.get_catalog_info_pointer(catalog_base_dir) - catalog_info = BaseCatalogInfo.read_from_metadata_file(catalog_info_path) - return catalog_info.catalog_type + return TableProperties.read_from_dir(catalog_base_path).catalog_type def _get_loader_from_catalog_type(catalog_type: CatalogType) -> Type[Dataset]: diff --git a/tests/conftest.py b/tests/conftest.py index ff2588b4..134e223b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,3 @@ -import dataclasses import os.path from pathlib import Path from typing import List @@ -7,11 +6,8 @@ import pyarrow as pa import pytest -from hats.catalog.association_catalog.association_catalog_info import AssociationCatalogInfo from hats.catalog.association_catalog.partition_join_info import PartitionJoinInfo -from hats.catalog.catalog_info import CatalogInfo -from hats.catalog.dataset.base_catalog_info import BaseCatalogInfo -from hats.catalog.margin_cache import MarginCacheCatalogInfo +from hats.catalog.dataset.table_properties import TableProperties from hats.inspection.almanac import Almanac from hats.pixel_math import HealpixPixel @@ -51,27 +47,6 @@ def small_sky_source_object_index_dir(test_data_dir): return test_data_dir / SMALL_SKY_SOURCE_OBJECT_INDEX_DIR_NAME -@pytest.fixture -def assert_catalog_info_matches_dict(): - def assert_match(catalog_info: BaseCatalogInfo, dictionary: dict): - """Check that all members of the catalog_info object match dictionary - elements, where specified.""" - catalog_info_dict = dataclasses.asdict(catalog_info) - for key, value in dictionary.items(): - assert catalog_info_dict[key] == value - - return assert_match - - -@pytest.fixture -def base_catalog_info_data() -> dict: - return { - "catalog_name": "test_name", - "catalog_type": "object", - "total_rows": 10, - } - - @pytest.fixture def catalog_info_data() -> dict: return { @@ -84,6 +59,11 @@ def catalog_info_data() -> dict: } +@pytest.fixture +def catalog_info(catalog_info_data) -> TableProperties: + return TableProperties(**catalog_info_data) + + @pytest.fixture def association_catalog_info_data() -> dict: return { @@ -101,19 +81,12 @@ def association_catalog_info_data() -> dict: @pytest.fixture -def source_catalog_info() -> dict: - return { - "catalog_name": "test_source", - "catalog_type": "source", - "total_rows": 100, - "epoch": "J2000", - "ra_column": "source_ra", - "dec_column": "source_dec", - } +def association_catalog_info(association_catalog_info_data) -> TableProperties: + return TableProperties(**association_catalog_info_data) @pytest.fixture -def source_catalog_info_with_extra() -> dict: +def source_catalog_info() -> dict: return { "catalog_name": "test_source", "catalog_type": "source", @@ -121,11 +94,6 @@ def source_catalog_info_with_extra() -> dict: "epoch": "J2000", "ra_column": "source_ra", "dec_column": "source_dec", - "primary_catalog": "test_name", - "mjd_column": "mjd", - "band_column": "band", - "mag_column": "mag", - "mag_err_column": "", } @@ -144,26 +112,8 @@ def margin_cache_catalog_info_data() -> dict: @pytest.fixture -def index_catalog_info() -> dict: - return { - "catalog_name": "test_index", - "catalog_type": "index", - "total_rows": 100, - "primary_catalog": "test_name", - "indexing_column": "id", - } - - -@pytest.fixture -def index_catalog_info_with_extra() -> dict: - return { - "catalog_name": "test_index", - "catalog_type": "index", - "total_rows": 100, - "primary_catalog": "test_name", - "indexing_column": "id", - "extra_columns": ["foo", "bar"], - } +def margin_catalog_info(margin_cache_catalog_info_data) -> TableProperties: + return TableProperties(**margin_cache_catalog_info_data) @pytest.fixture @@ -241,36 +191,11 @@ def dataset_path(test_data_dir) -> str: return test_data_dir / "info_only" / "dataset" -@pytest.fixture -def base_catalog_info_file(dataset_path) -> str: - return dataset_path / "catalog_info.json" - - -@pytest.fixture -def base_catalog_info(base_catalog_info_data) -> BaseCatalogInfo: - return BaseCatalogInfo(**base_catalog_info_data) - - @pytest.fixture def catalog_path(test_data_dir) -> str: return test_data_dir / "info_only" / "catalog" -@pytest.fixture -def catalog_info_file(catalog_path) -> str: - return catalog_path / "catalog_info.json" - - -@pytest.fixture -def catalog_info(catalog_info_data) -> CatalogInfo: - return CatalogInfo(**catalog_info_data) - - -@pytest.fixture -def margin_catalog_info(margin_cache_catalog_info_data) -> MarginCacheCatalogInfo: - return MarginCacheCatalogInfo(**margin_cache_catalog_info_data) - - @pytest.fixture def margin_catalog_pixels() -> List[HealpixPixel]: return [ @@ -297,26 +222,6 @@ def association_catalog_path(test_data_dir) -> str: return test_data_dir / "small_sky_to_small_sky_order1" -@pytest.fixture -def association_catalog_info_file(association_catalog_path) -> str: - return association_catalog_path / "catalog_info.json" - - -@pytest.fixture -def index_catalog_info_file(test_data_dir) -> str: - return test_data_dir / "info_only" / "index_catalog" / "catalog_info.json" - - -@pytest.fixture -def margin_cache_catalog_info_file(test_data_dir) -> str: - return test_data_dir / "info_only" / "margin_cache" / "catalog_info.json" - - -@pytest.fixture -def source_catalog_info_file(test_data_dir) -> str: - return test_data_dir / "small_sky_source" / "catalog_info.json" - - @pytest.fixture def small_sky_source_dir(test_data_dir) -> str: return test_data_dir / "small_sky_source" @@ -343,11 +248,6 @@ def small_sky_source_pixels(): ] -@pytest.fixture -def association_catalog_info(association_catalog_info_data) -> AssociationCatalogInfo: - return AssociationCatalogInfo(**association_catalog_info_data) - - @pytest.fixture def association_catalog_partition_join_file(association_catalog_path) -> str: return association_catalog_path / "partition_join_info.csv" diff --git a/tests/data/info_only/catalog/catalog_info.json b/tests/data/info_only/catalog/catalog_info.json deleted file mode 100644 index cb1eb03f..00000000 --- a/tests/data/info_only/catalog/catalog_info.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "catalog_name": "catalog", - "catalog_type": "object", - "epoch": "J2000", - "ra_column": "ra", - "dec_column": "dec", - "total_rows": 10 -} \ No newline at end of file diff --git a/tests/data/info_only/catalog/properties b/tests/data/info_only/catalog/properties new file mode 100644 index 00000000..83fef746 --- /dev/null +++ b/tests/data/info_only/catalog/properties @@ -0,0 +1,7 @@ +obs_collection = catalog +dataproduct_type = object +hats_col_j2000_ra = ra +hats_col_j2000_dec = dec +hats_max_rows = 1000 +hats_nrows = 10 +hats_order = 0 \ No newline at end of file diff --git a/tests/data/info_only/dataset/catalog_info.json b/tests/data/info_only/dataset/catalog_info.json deleted file mode 100644 index 229bb6db..00000000 --- a/tests/data/info_only/dataset/catalog_info.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "catalog_name": "dataset", - "catalog_type": "object", - "total_rows": 10 -} \ No newline at end of file diff --git a/tests/data/info_only/dataset/properties b/tests/data/info_only/dataset/properties new file mode 100644 index 00000000..e7808ef3 --- /dev/null +++ b/tests/data/info_only/dataset/properties @@ -0,0 +1,7 @@ +obs_collection = dataset +dataproduct_type = object +hats_col_j2000_ra = ra +hats_col_j2000_dec = dec +hats_max_rows = 1000 +hats_nrows = 10 +hats_order = 0 \ No newline at end of file diff --git a/tests/data/info_only/index_catalog/catalog_info.json b/tests/data/info_only/index_catalog/catalog_info.json deleted file mode 100644 index fb824751..00000000 --- a/tests/data/info_only/index_catalog/catalog_info.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "catalog_name": "index_catalog", - "catalog_type": "index", - "total_rows": 100, - "primary_catalog": "catalog", - "indexing_column": "id" -} \ No newline at end of file diff --git a/tests/data/info_only/index_catalog/properties b/tests/data/info_only/index_catalog/properties new file mode 100644 index 00000000..5a62e29a --- /dev/null +++ b/tests/data/info_only/index_catalog/properties @@ -0,0 +1,7 @@ +obs_collection = index_catalog +dataproduct_type = index +hats_max_rows = 1000 +hats_nrows = 100 +hats_order = 0 +hats_index_column=id +hats_primary_table_url=catalog \ No newline at end of file diff --git a/tests/data/info_only/margin_cache/catalog_info.json b/tests/data/info_only/margin_cache/catalog_info.json deleted file mode 100644 index d87cfb9b..00000000 --- a/tests/data/info_only/margin_cache/catalog_info.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "catalog_name": "margin_cache", - "catalog_type": "margin", - "total_rows": 100, - "epoch": "J2000", - "ra_column": "ra", - "dec_column": "dec", - "primary_catalog": "catalog", - "margin_threshold": 0.5 -} \ No newline at end of file diff --git a/tests/data/info_only/margin_cache/properties b/tests/data/info_only/margin_cache/properties new file mode 100644 index 00000000..7c0b5177 --- /dev/null +++ b/tests/data/info_only/margin_cache/properties @@ -0,0 +1,9 @@ +obs_collection = margin_cache +dataproduct_type = margin +hats_col_j2000_ra = ra +hats_col_j2000_dec = dec +hats_max_rows = 1000 +hats_nrows = 100 +hats_order = 0 +hats_margin_threshold=0.5 +hats_primary_table_url=catalog \ No newline at end of file diff --git a/tests/data/small_sky/catalog_info.json b/tests/data/small_sky/catalog_info.json deleted file mode 100644 index b3fd9a27..00000000 --- a/tests/data/small_sky/catalog_info.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "catalog_name": "small_sky", - "catalog_type": "object", - "total_rows": 131, - "epoch": "J2000", - "ra_column": "ra", - "dec_column": "dec" -} diff --git a/tests/data/small_sky/properties b/tests/data/small_sky/properties new file mode 100644 index 00000000..63178214 --- /dev/null +++ b/tests/data/small_sky/properties @@ -0,0 +1,7 @@ +obs_collection = small_sky +dataproduct_type = object +hats_col_j2000_ra = ra +hats_col_j2000_dec = dec +hats_max_rows = 1000 +hats_nrows = 131 +hats_order = 0 \ No newline at end of file diff --git a/tests/data/small_sky/provenance_info.json b/tests/data/small_sky/provenance_info.json deleted file mode 100644 index 9beaa898..00000000 --- a/tests/data/small_sky/provenance_info.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "catalog_name": "small_sky", - "catalog_type": "object", - "total_rows": 131, - "epoch": "J2000", - "ra_column": "ra", - "dec_column": "dec", - "version": "0.3.10.dev6+g5cb658f", - "generation_date": "2024.09.20", - "tool_args": { - "tool_name": "hats_import", - "version": "0.3.6.dev20+gc573604", - "runtime_args": { - "catalog_name": "small_sky", - "output_path": ".", - "output_artifact_name": "small_sky", - "tmp_dir": "/tmp/tmpcczp4su4", - "dask_tmp": null, - "dask_n_workers": 1, - "dask_threads_per_worker": 1, - "catalog_path": "small_sky", - "tmp_path": "/tmp/tmpcczp4su4/small_sky/intermediate", - "epoch": "J2000", - "catalog_type": "object", - "input_path": "../../../hipscat-import/tests/data/small_sky", - "input_paths": [ - "../../../hipscat-import/tests/data/small_sky/catalog.csv" - ], - "input_file_list": [], - "ra_column": "ra", - "dec_column": "dec", - "use_healpix_29": false, - "sort_columns": null, - "constant_healpix_order": -1, - "lowest_healpix_order": 0, - "highest_healpix_order": 10, - "pixel_threshold": 1000000, - "mapping_healpix_order": 10, - "debug_stats_only": false, - "file_reader_info": { - "input_reader_type": "CsvReader", - "chunksize": 500000, - "header": "infer", - "schema_file": null, - "column_names": null, - "type_map": null, - "parquet_kwargs": null, - "upath_kwargs": null, - "kwargs": {} - } - } - } -} diff --git a/tests/data/small_sky_order1/catalog_info.json b/tests/data/small_sky_order1/catalog_info.json deleted file mode 100644 index a61f882a..00000000 --- a/tests/data/small_sky_order1/catalog_info.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "catalog_name": "small_sky_order1", - "catalog_type": "object", - "total_rows": 131, - "epoch": "J2000", - "ra_column": "ra", - "dec_column": "dec" -} diff --git a/tests/data/small_sky_order1/properties b/tests/data/small_sky_order1/properties new file mode 100644 index 00000000..5211d605 --- /dev/null +++ b/tests/data/small_sky_order1/properties @@ -0,0 +1,7 @@ +obs_collection = small_sky_order1 +dataproduct_type = object +hats_col_j2000_ra = ra +hats_col_j2000_dec = dec +hats_max_rows = 1000 +hats_nrows = 131 +hats_order = 0 \ No newline at end of file diff --git a/tests/data/small_sky_order1/provenance_info.json b/tests/data/small_sky_order1/provenance_info.json deleted file mode 100644 index 1e67528f..00000000 --- a/tests/data/small_sky_order1/provenance_info.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "catalog_name": "small_sky_order1", - "catalog_type": "object", - "total_rows": 131, - "epoch": "J2000", - "ra_column": "ra", - "dec_column": "dec", - "version": "0.3.10.dev6+g5cb658f", - "generation_date": "2024.09.20", - "tool_args": { - "tool_name": "hats_import", - "version": "0.3.6.dev20+gc573604", - "runtime_args": { - "catalog_name": "small_sky_order1", - "output_path": ".", - "output_artifact_name": "small_sky_order1", - "tmp_dir": "/tmp/tmpnu_b__vk", - "dask_tmp": null, - "dask_n_workers": 1, - "dask_threads_per_worker": 1, - "catalog_path": "small_sky_order1", - "tmp_path": "/tmp/tmpnu_b__vk/small_sky_order1/intermediate", - "epoch": "J2000", - "catalog_type": "object", - "input_path": "../../../hipscat-import/tests/data/small_sky", - "input_paths": [ - "../../../hipscat-import/tests/data/small_sky/catalog.csv" - ], - "input_file_list": [], - "ra_column": "ra", - "dec_column": "dec", - "use_healpix_29": false, - "sort_columns": null, - "constant_healpix_order": 1, - "lowest_healpix_order": 0, - "highest_healpix_order": 10, - "pixel_threshold": 1000000, - "mapping_healpix_order": 1, - "debug_stats_only": false, - "file_reader_info": { - "input_reader_type": "CsvReader", - "chunksize": 500000, - "header": "infer", - "schema_file": null, - "column_names": null, - "type_map": null, - "parquet_kwargs": null, - "upath_kwargs": null, - "kwargs": {} - } - } - } -} diff --git a/tests/data/small_sky_order1_id_index/catalog_info.json b/tests/data/small_sky_order1_id_index/catalog_info.json deleted file mode 100644 index de300080..00000000 --- a/tests/data/small_sky_order1_id_index/catalog_info.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "catalog_name": "small_sky_order1_id_index", - "catalog_type": "index", - "total_rows": 131, - "primary_catalog": "small_sky", - "indexing_column": "id", - "extra_columns": [] -} diff --git a/tests/data/small_sky_order1_id_index/properties b/tests/data/small_sky_order1_id_index/properties new file mode 100644 index 00000000..7ad9bb76 --- /dev/null +++ b/tests/data/small_sky_order1_id_index/properties @@ -0,0 +1,7 @@ +obs_collection = small_sky_order1_id_index +dataproduct_type = index +hats_max_rows = 1000 +hats_nrows = 131 +hats_order = 0 +hats_index_column=id +hats_primary_table_url=small_sky \ No newline at end of file diff --git a/tests/data/small_sky_order1_id_index/provenance_info.json b/tests/data/small_sky_order1_id_index/provenance_info.json deleted file mode 100644 index add01c76..00000000 --- a/tests/data/small_sky_order1_id_index/provenance_info.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "catalog_name": "small_sky_order1_id_index", - "catalog_type": "index", - "total_rows": 131, - "primary_catalog": "small_sky", - "indexing_column": "id", - "extra_columns": [], - "version": "0.3.10.dev6+g5cb658f", - "generation_date": "2024.09.20", - "tool_args": { - "tool_name": "hats_import", - "version": "0.3.6.dev20+gc573604", - "runtime_args": { - "catalog_name": "small_sky_order1_id_index", - "output_path": ".", - "output_artifact_name": "small_sky_order1_id_index", - "tmp_dir": "/tmp/tmp0nyfm4am", - "dask_tmp": null, - "dask_n_workers": 1, - "dask_threads_per_worker": 1, - "catalog_path": "small_sky_order1_id_index", - "tmp_path": "/tmp/tmp0nyfm4am/small_sky_order1_id_index/intermediate", - "input_catalog_path": "small_sky", - "indexing_column": "id", - "extra_columns": [], - "include_healpix_29": false, - "include_order_pixel": true, - "include_radec": false - } - } -} diff --git a/tests/data/small_sky_order1_margin/catalog_info.json b/tests/data/small_sky_order1_margin/catalog_info.json deleted file mode 100644 index f27aa6ae..00000000 --- a/tests/data/small_sky_order1_margin/catalog_info.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "catalog_name": "small_sky_order1_margin", - "catalog_type": "margin", - "total_rows": 28, - "epoch": "J2000", - "ra_column": "ra", - "dec_column": "dec", - "primary_catalog": "small_sky_order1", - "margin_threshold": 7200 -} diff --git a/tests/data/small_sky_order1_margin/properties b/tests/data/small_sky_order1_margin/properties new file mode 100644 index 00000000..2d4f6264 --- /dev/null +++ b/tests/data/small_sky_order1_margin/properties @@ -0,0 +1,9 @@ +obs_collection = small_sky_order1_margin +dataproduct_type = margin +hats_col_j2000_ra = ra +hats_col_j2000_dec = dec +hats_max_rows = 1000 +hats_nrows = 28 +hats_order = 0 +hats_primary_table_url=small_sky_order1 +hats_margin_threshold=7200 \ No newline at end of file diff --git a/tests/data/small_sky_order1_margin/provenance_info.json b/tests/data/small_sky_order1_margin/provenance_info.json deleted file mode 100644 index 639f80e9..00000000 --- a/tests/data/small_sky_order1_margin/provenance_info.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "catalog_name": "small_sky_order1_margin", - "catalog_type": "margin", - "total_rows": 28, - "epoch": "J2000", - "ra_column": "ra", - "dec_column": "dec", - "primary_catalog": "small_sky_order1", - "margin_threshold": 7200, - "version": "0.3.10.dev6+g5cb658f", - "generation_date": "2024.09.20", - "tool_args": { - "tool_name": "hats_import", - "version": "0.3.6.dev20+gc573604", - "runtime_args": { - "catalog_name": "small_sky_order1_margin", - "output_path": ".", - "output_artifact_name": "small_sky_order1_margin", - "tmp_dir": "/tmp/tmpi4dfahpr", - "dask_tmp": null, - "dask_n_workers": 1, - "dask_threads_per_worker": 1, - "catalog_path": "small_sky_order1_margin", - "tmp_path": "/tmp/tmpi4dfahpr/small_sky_order1_margin/intermediate", - "input_catalog_path": "small_sky_order1", - "margin_threshold": 7200, - "margin_order": 4, - "debug_filter_pixel_list": [] - } - } -} diff --git a/tests/data/small_sky_source/catalog_info.json b/tests/data/small_sky_source/catalog_info.json deleted file mode 100644 index 97e68711..00000000 --- a/tests/data/small_sky_source/catalog_info.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "catalog_name": "small_sky_source", - "catalog_type": "source", - "total_rows": 17161, - "epoch": "J2000", - "ra_column": "source_ra", - "dec_column": "source_dec" -} diff --git a/tests/data/small_sky_source/properties b/tests/data/small_sky_source/properties new file mode 100644 index 00000000..9cf00cca --- /dev/null +++ b/tests/data/small_sky_source/properties @@ -0,0 +1,7 @@ +obs_collection = small_sky_source +dataproduct_type = source +hats_col_j2000_ra = source_ra +hats_col_j2000_dec = source_dec +hats_max_rows = 1000 +hats_nrows = 17161 +hats_order = 0 \ No newline at end of file diff --git a/tests/data/small_sky_source/provenance_info.json b/tests/data/small_sky_source/provenance_info.json deleted file mode 100644 index 5998439c..00000000 --- a/tests/data/small_sky_source/provenance_info.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "catalog_name": "small_sky_source", - "catalog_type": "source", - "total_rows": 17161, - "epoch": "J2000", - "ra_column": "source_ra", - "dec_column": "source_dec", - "version": "0.3.10.dev6+g5cb658f", - "generation_date": "2024.09.20", - "tool_args": { - "tool_name": "hats_import", - "version": "0.3.6.dev20+gc573604", - "runtime_args": { - "catalog_name": "small_sky_source", - "output_path": ".", - "output_artifact_name": "small_sky_source", - "tmp_dir": "/tmp/tmp6vxk1tuz", - "dask_tmp": null, - "dask_n_workers": 1, - "dask_threads_per_worker": 1, - "catalog_path": "small_sky_source", - "tmp_path": "/tmp/tmp6vxk1tuz/small_sky_source/intermediate", - "epoch": "J2000", - "catalog_type": "source", - "input_path": "../../../hipscat-import/tests/data/small_sky_source", - "input_paths": [ - "../../../hipscat-import/tests/data/small_sky_source/small_sky_source.csv" - ], - "input_file_list": [], - "ra_column": "source_ra", - "dec_column": "source_dec", - "use_healpix_29": false, - "sort_columns": null, - "constant_healpix_order": -1, - "lowest_healpix_order": 0, - "highest_healpix_order": 10, - "pixel_threshold": 3000, - "mapping_healpix_order": 10, - "debug_stats_only": false, - "file_reader_info": { - "input_reader_type": "CsvReader", - "chunksize": 500000, - "header": "infer", - "schema_file": null, - "column_names": null, - "type_map": null, - "parquet_kwargs": null, - "upath_kwargs": null, - "kwargs": {} - } - } - } -} diff --git a/tests/data/small_sky_source_object_index/catalog_info.json b/tests/data/small_sky_source_object_index/catalog_info.json deleted file mode 100644 index ab2d3aa0..00000000 --- a/tests/data/small_sky_source_object_index/catalog_info.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "catalog_name": "small_sky_source_object_index", - "catalog_type": "index", - "total_rows": 148, - "primary_catalog": "small_sky_source", - "indexing_column": "object_id", - "extra_columns": [] -} diff --git a/tests/data/small_sky_source_object_index/properties b/tests/data/small_sky_source_object_index/properties new file mode 100644 index 00000000..136c9a80 --- /dev/null +++ b/tests/data/small_sky_source_object_index/properties @@ -0,0 +1,7 @@ +obs_collection = small_sky_source_object_index +dataproduct_type = index +hats_max_rows = 1000 +hats_nrows = 148 +hats_order = 0 +hats_index_column=object_id +hats_primary_table_url=small_sky_source \ No newline at end of file diff --git a/tests/data/small_sky_source_object_index/provenance_info.json b/tests/data/small_sky_source_object_index/provenance_info.json deleted file mode 100644 index 72fe842c..00000000 --- a/tests/data/small_sky_source_object_index/provenance_info.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "catalog_name": "small_sky_source_object_index", - "catalog_type": "index", - "total_rows": 148, - "primary_catalog": "small_sky_source", - "indexing_column": "object_id", - "extra_columns": [], - "version": "0.3.10.dev6+g5cb658f", - "generation_date": "2024.09.20", - "tool_args": { - "tool_name": "hats_import", - "version": "0.3.6.dev20+gc573604", - "runtime_args": { - "catalog_name": "small_sky_source_object_index", - "output_path": ".", - "output_artifact_name": "small_sky_source_object_index", - "tmp_dir": "/tmp/tmpde8h2vjg", - "dask_tmp": null, - "dask_n_workers": 1, - "dask_threads_per_worker": 1, - "catalog_path": "small_sky_source_object_index", - "tmp_path": "/tmp/tmpde8h2vjg/small_sky_source_object_index/intermediate", - "input_catalog_path": "small_sky_source", - "indexing_column": "object_id", - "extra_columns": [], - "include_healpix_29": false, - "include_order_pixel": true, - "include_radec": false - } - } -} diff --git a/tests/data/small_sky_to_small_sky_order1/catalog_info.json b/tests/data/small_sky_to_small_sky_order1/catalog_info.json deleted file mode 100644 index f739cf61..00000000 --- a/tests/data/small_sky_to_small_sky_order1/catalog_info.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "catalog_name": "small_sky_to_small_sky_order1", - "catalog_type": "association", - "primary_catalog": "small_sky", - "primary_column": "id", - "primary_column_association": "id_small_sky", - "join_catalog": "small_sky_order1", - "join_column": "id", - "join_column_association": "id_small_sky_order1", - "contains_leaf_files": false, - "total_rows": 131 -} \ No newline at end of file diff --git a/tests/data/small_sky_to_small_sky_order1/properties b/tests/data/small_sky_to_small_sky_order1/properties new file mode 100644 index 00000000..0dad729b --- /dev/null +++ b/tests/data/small_sky_to_small_sky_order1/properties @@ -0,0 +1,10 @@ +obs_collection = small_sky_to_small_sky_order1 +dataproduct_type = association +hats_nrows = 131 +hats_assn_join_table_url=small_sky_order1 +hats_assn_leaf_files=false +hats_col_assn_join=id +hats_col_assn_join_assn=id_small_sky_order1 +hats_col_assn_primary=id +hats_col_assn_primary_assn=id_small_sky +hats_primary_table_url=small_sky \ No newline at end of file diff --git a/tests/hats/catalog/association_catalog/test_association_catalog.py b/tests/hats/catalog/association_catalog/test_association_catalog.py index a7e2dad1..c19bc675 100644 --- a/tests/hats/catalog/association_catalog/test_association_catalog.py +++ b/tests/hats/catalog/association_catalog/test_association_catalog.py @@ -1,13 +1,12 @@ -import json import os import pandas as pd import pyarrow as pa import pytest -from hats.catalog import CatalogType from hats.catalog.association_catalog.association_catalog import AssociationCatalog from hats.catalog.association_catalog.partition_join_info import PartitionJoinInfo +from hats.catalog.dataset.table_properties import TableProperties from hats.loaders import read_hats from hats.pixel_math import HealpixPixel @@ -27,18 +26,6 @@ def test_init_catalog(association_catalog_info, association_catalog_join_pixels) assert hp_pixel in catalog.pixel_tree -def test_wrong_catalog_type(association_catalog_info, association_catalog_join_pixels): - association_catalog_info.catalog_type = CatalogType.OBJECT - with pytest.raises(ValueError, match="catalog_type"): - AssociationCatalog(association_catalog_info, [HealpixPixel(0, 11)], association_catalog_join_pixels) - - -def test_wrong_catalog_info_type(catalog_info, association_catalog_join_pixels): - catalog_info.catalog_type = CatalogType.ASSOCIATION - with pytest.raises(TypeError, match="catalog_info"): - AssociationCatalog(catalog_info, [HealpixPixel(0, 11)], association_catalog_join_pixels) - - def test_wrong_join_pixels_type(association_catalog_info): with pytest.raises(TypeError, match="join_pixels"): AssociationCatalog(association_catalog_info, [HealpixPixel(0, 11)], "test") @@ -83,13 +70,12 @@ def test_empty_directory(tmp_path, association_catalog_info_data, association_ca os.makedirs(catalog_path, exist_ok=True) ## Path exists but there's nothing there - with pytest.raises(FileNotFoundError, match="catalog info"): + with pytest.raises(FileNotFoundError, match="properties file"): AssociationCatalog.read_hats(catalog_path) ## catalog_info file exists - getting closer - file_name = catalog_path / "catalog_info.json" - with open(file_name, "w", encoding="utf-8") as metadata_file: - metadata_file.write(json.dumps(association_catalog_info_data)) + properties = TableProperties(**association_catalog_info_data) + properties.to_properties_file(catalog_path) with pytest.raises(FileNotFoundError, match="metadata"): read_hats(catalog_path) @@ -110,9 +96,8 @@ def test_csv_round_trip(tmp_path, association_catalog_info_data, association_cat catalog_path = tmp_path / "empty" os.makedirs(catalog_path, exist_ok=True) - file_name = catalog_path / "catalog_info.json" - with open(file_name, "w", encoding="utf-8") as metadata_file: - metadata_file.write(json.dumps(association_catalog_info_data)) + properties = TableProperties(**association_catalog_info_data) + properties.to_properties_file(catalog_path) with pytest.raises(FileNotFoundError, match="partition"): read_hats(catalog_path) @@ -120,7 +105,7 @@ def test_csv_round_trip(tmp_path, association_catalog_info_data, association_cat file_name = catalog_path / "partition_info.csv" with open(file_name, "w", encoding="utf-8") as metadata_file: # dump some garbage in there - just needs to exist. - metadata_file.write(json.dumps(association_catalog_info_data)) + metadata_file.write("Norder,Npix") with pytest.raises(FileNotFoundError, match="partition"): read_hats(catalog_path) diff --git a/tests/hats/catalog/association_catalog/test_association_catalog_info.py b/tests/hats/catalog/association_catalog/test_association_catalog_info.py deleted file mode 100644 index 4f112256..00000000 --- a/tests/hats/catalog/association_catalog/test_association_catalog_info.py +++ /dev/null @@ -1,66 +0,0 @@ -import dataclasses -import json - -import pytest - -from hats.catalog.association_catalog.association_catalog_info import AssociationCatalogInfo -from hats.catalog.catalog_type import CatalogType - - -def test_association_catalog_info(association_catalog_info_data, assert_catalog_info_matches_dict): - info = AssociationCatalogInfo(**association_catalog_info_data) - assert_catalog_info_matches_dict(info, association_catalog_info_data) - - -def test_str(association_catalog_info_data): - correct_string = "" - for name, value in association_catalog_info_data.items(): - correct_string += f" {name} {value}\n" - cat_info = AssociationCatalogInfo(**association_catalog_info_data) - assert str(cat_info) == correct_string - - -def test_read_from_file(association_catalog_info_file, assert_catalog_info_matches_dict): - catalog_info = AssociationCatalogInfo.read_from_metadata_file(association_catalog_info_file) - for column in [ - "catalog_name", - "catalog_type", - "total_rows", - "primary_column", - "primary_catalog", - "join_column", - "join_catalog", - ]: - assert column in dataclasses.asdict(catalog_info) - - with open(association_catalog_info_file, "r", encoding="utf-8") as cat_info_file: - catalog_info_json = json.load(cat_info_file) - assert_catalog_info_matches_dict(catalog_info, catalog_info_json) - - -def test_required_fields_missing(association_catalog_info_data): - required_fields = ["primary_catalog", "join_catalog"] - for required_field in required_fields: - assert required_field in AssociationCatalogInfo.required_fields - for field in required_fields: - init_data = association_catalog_info_data.copy() - init_data[field] = None - with pytest.raises(ValueError, match=field): - AssociationCatalogInfo(**init_data) - - -def test_type_missing(association_catalog_info_data): - init_data = association_catalog_info_data.copy() - init_data["catalog_type"] = None - catalog_info = AssociationCatalogInfo(**init_data) - assert catalog_info.catalog_type == CatalogType.ASSOCIATION - - -def test_wrong_type(association_catalog_info_data, catalog_info_data): - with pytest.raises(TypeError, match="unexpected"): - AssociationCatalogInfo(**catalog_info_data) - - with pytest.raises(ValueError, match=f"{CatalogType.ASSOCIATION}"): - init_data = association_catalog_info_data.copy() - init_data["catalog_type"] = CatalogType.OBJECT - AssociationCatalogInfo(**init_data) diff --git a/tests/hats/catalog/dataset/test_base_catalog_info.py b/tests/hats/catalog/dataset/test_base_catalog_info.py deleted file mode 100644 index 248acaed..00000000 --- a/tests/hats/catalog/dataset/test_base_catalog_info.py +++ /dev/null @@ -1,39 +0,0 @@ -import json - -import pytest - -from hats.catalog.dataset.base_catalog_info import BaseCatalogInfo - - -def test_fields_init(base_catalog_info_data, assert_catalog_info_matches_dict): - catalog_info = BaseCatalogInfo(**base_catalog_info_data) - assert_catalog_info_matches_dict(catalog_info, base_catalog_info_data) - - -def test_required_fields_missing(base_catalog_info_data): - for field in BaseCatalogInfo.required_fields: - init_data = base_catalog_info_data.copy() - init_data[field] = None - with pytest.raises(ValueError, match=field): - BaseCatalogInfo(**init_data) - - -def test_wrong_catalog_type(base_catalog_info_data): - base_catalog_info_data["catalog_type"] = "wrong_type" - with pytest.raises(ValueError, match=base_catalog_info_data["catalog_type"]): - BaseCatalogInfo(**base_catalog_info_data) - - -def test_str(base_catalog_info_data): - correct_string = "" - for name, value in base_catalog_info_data.items(): - correct_string += f" {name} {value}\n" - cat_info = BaseCatalogInfo(**base_catalog_info_data) - assert str(cat_info) == correct_string - - -def test_read_from_file(base_catalog_info_file, assert_catalog_info_matches_dict): - catalog_info = BaseCatalogInfo.read_from_metadata_file(base_catalog_info_file) - with open(base_catalog_info_file, "r", encoding="utf-8") as cat_info_file: - catalog_info_json = json.load(cat_info_file) - assert_catalog_info_matches_dict(catalog_info, catalog_info_json) diff --git a/tests/hats/catalog/dataset/test_catalog_info_factory.py b/tests/hats/catalog/dataset/test_catalog_info_factory.py deleted file mode 100644 index 64edd6f2..00000000 --- a/tests/hats/catalog/dataset/test_catalog_info_factory.py +++ /dev/null @@ -1,109 +0,0 @@ -import pytest - -from hats.catalog.association_catalog.association_catalog_info import AssociationCatalogInfo -from hats.catalog.catalog_info import CatalogInfo -from hats.catalog.dataset.base_catalog_info import BaseCatalogInfo -from hats.catalog.dataset.catalog_info_factory import create_catalog_info, from_catalog_dir -from hats.catalog.index.index_catalog_info import IndexCatalogInfo -from hats.catalog.margin_cache.margin_cache_catalog_info import MarginCacheCatalogInfo -from hats.catalog.source_catalog.source_catalog_info import SourceCatalogInfo - - -def test_create_catalog_info_bad(base_catalog_info_data): - base_catalog_info_data["catalog_type"] = "foo" - with pytest.raises(ValueError, match="Unknown"): - create_catalog_info(base_catalog_info_data) - - base_catalog_info_data.pop("catalog_type") - with pytest.raises(ValueError, match="required"): - create_catalog_info(base_catalog_info_data) - - -def test_create_catalog_info(base_catalog_info_data, catalog_info_data): - catalog_info = create_catalog_info(base_catalog_info_data) - assert catalog_info.catalog_name == "test_name" - assert isinstance(catalog_info, BaseCatalogInfo) - assert isinstance(catalog_info, CatalogInfo) - - catalog_info = create_catalog_info(catalog_info_data) - assert catalog_info.catalog_name == "test_name" - assert isinstance(catalog_info, BaseCatalogInfo) - assert isinstance(catalog_info, CatalogInfo) - - -def test_create_catalog_info_association(association_catalog_info_data): - catalog_info = create_catalog_info(association_catalog_info_data) - assert catalog_info.catalog_name == "test_name" - assert isinstance(catalog_info, BaseCatalogInfo) - assert isinstance(catalog_info, AssociationCatalogInfo) - - -def test_create_catalog_info_source(source_catalog_info, source_catalog_info_with_extra): - catalog_info = create_catalog_info(source_catalog_info) - assert catalog_info.catalog_name == "test_source" - assert isinstance(catalog_info, BaseCatalogInfo) - assert isinstance(catalog_info, SourceCatalogInfo) - - catalog_info = create_catalog_info(source_catalog_info_with_extra) - assert catalog_info.catalog_name == "test_source" - assert isinstance(catalog_info, BaseCatalogInfo) - assert isinstance(catalog_info, SourceCatalogInfo) - - -def test_create_catalog_info_margin_cache(margin_cache_catalog_info_data): - catalog_info = create_catalog_info(margin_cache_catalog_info_data) - assert catalog_info.catalog_name == "test_margin" - assert isinstance(catalog_info, BaseCatalogInfo) - assert isinstance(catalog_info, MarginCacheCatalogInfo) - - -def test_create_catalog_info_index(index_catalog_info, index_catalog_info_with_extra): - catalog_info = create_catalog_info(index_catalog_info) - assert catalog_info.catalog_name == "test_index" - assert isinstance(catalog_info, BaseCatalogInfo) - assert isinstance(catalog_info, IndexCatalogInfo) - - catalog_info = create_catalog_info(index_catalog_info_with_extra) - assert catalog_info.catalog_name == "test_index" - assert isinstance(catalog_info, BaseCatalogInfo) - assert isinstance(catalog_info, IndexCatalogInfo) - - -def test_from_catalog_dir_object(small_sky_dir): - catalog_info = from_catalog_dir(small_sky_dir) - assert catalog_info.catalog_name == "small_sky" - assert isinstance(catalog_info, BaseCatalogInfo) - assert isinstance(catalog_info, CatalogInfo) - - -def test_from_catalog_dir_source(source_catalog_info_file): - catalog_info = from_catalog_dir(source_catalog_info_file) - assert catalog_info.catalog_name == "small_sky_source" - assert isinstance(catalog_info, BaseCatalogInfo) - assert isinstance(catalog_info, SourceCatalogInfo) - - -def test_from_catalog_dir_margin_cache(margin_cache_catalog_info_file): - catalog_info = from_catalog_dir(margin_cache_catalog_info_file) - assert catalog_info.catalog_name == "margin_cache" - assert isinstance(catalog_info, BaseCatalogInfo) - assert isinstance(catalog_info, MarginCacheCatalogInfo) - - -def test_from_catalog_dir_index(index_catalog_info_file): - catalog_info = from_catalog_dir(index_catalog_info_file) - assert catalog_info.catalog_name == "index_catalog" - assert isinstance(catalog_info, BaseCatalogInfo) - assert isinstance(catalog_info, IndexCatalogInfo) - - -def test_from_catalog_dir_association(association_catalog_path, association_catalog_info_file): - catalog_info = from_catalog_dir(association_catalog_path) - assert catalog_info.catalog_name == "small_sky_to_small_sky_order1" - assert isinstance(catalog_info, BaseCatalogInfo) - assert isinstance(catalog_info, AssociationCatalogInfo) - - catalog_info = from_catalog_dir(association_catalog_info_file) - assert catalog_info.catalog_name == "small_sky_to_small_sky_order1" - assert isinstance(catalog_info, BaseCatalogInfo) - assert isinstance(catalog_info, AssociationCatalogInfo) diff --git a/tests/hats/catalog/dataset/test_dataset.py b/tests/hats/catalog/dataset/test_dataset.py index 823f14af..c68f6215 100644 --- a/tests/hats/catalog/dataset/test_dataset.py +++ b/tests/hats/catalog/dataset/test_dataset.py @@ -1,4 +1,3 @@ -import json import os import pytest @@ -6,28 +5,11 @@ from hats.catalog.dataset.dataset import Dataset -def test_dataset_init(base_catalog_info): - dataset = Dataset(base_catalog_info) - assert dataset.catalog_info == base_catalog_info - assert dataset.catalog_name == base_catalog_info.catalog_name - assert not dataset.on_disk - assert dataset.catalog_path is None - assert dataset.catalog_base_dir is None - - -def test_dataset_wrong_catalog_info(base_catalog_info_data): - with pytest.raises(TypeError, match="catalog_info"): - Dataset(base_catalog_info_data) - - -def test_read_hats(dataset_path, base_catalog_info_file, assert_catalog_info_matches_dict): +def test_read_hats(dataset_path): dataset = Dataset.read_hats(dataset_path) assert dataset.on_disk assert str(dataset.catalog_path) == str(dataset_path) assert str(dataset.catalog_base_dir) == str(dataset_path) - with open(base_catalog_info_file, "r", encoding="utf-8") as cat_info_file: - catalog_info_json = json.load(cat_info_file) - assert_catalog_info_matches_dict(dataset.catalog_info, catalog_info_json) def test_read_from_missing_folder(tmp_path): @@ -39,5 +21,5 @@ def test_read_from_missing_folder(tmp_path): def test_read_from_empty_folder(tmp_path): dataset_path = tmp_path / "dat" os.makedirs(dataset_path) - with pytest.raises(FileNotFoundError, match="catalog info"): + with pytest.raises(FileNotFoundError, match="properties file"): Dataset.read_hats(dataset_path) diff --git a/tests/hats/catalog/dataset/test_table_properties.py b/tests/hats/catalog/dataset/test_table_properties.py new file mode 100644 index 00000000..1c9bb407 --- /dev/null +++ b/tests/hats/catalog/dataset/test_table_properties.py @@ -0,0 +1,27 @@ +from hats.catalog.dataset.table_properties import TableProperties + + +def test_read_from_file_round_trip(dataset_path, tmp_path): + table_properties = TableProperties.read_from_dir(dataset_path) + + table_properties.to_properties_file(tmp_path) + + round_trip_properties = TableProperties.read_from_dir(tmp_path) + + assert table_properties == round_trip_properties + + +def test_read_from_dir_branches( + small_sky_dir, + small_sky_order1_dir, + association_catalog_path, + small_sky_source_object_index_dir, + margin_catalog_path, + small_sky_source_dir, +): + TableProperties.read_from_dir(small_sky_dir) + TableProperties.read_from_dir(small_sky_order1_dir) + TableProperties.read_from_dir(association_catalog_path) + TableProperties.read_from_dir(small_sky_source_object_index_dir) + TableProperties.read_from_dir(margin_catalog_path) + TableProperties.read_from_dir(small_sky_source_dir) diff --git a/tests/hats/catalog/index/test_index_catalog_info.py b/tests/hats/catalog/index/test_index_catalog_info.py deleted file mode 100644 index b8954203..00000000 --- a/tests/hats/catalog/index/test_index_catalog_info.py +++ /dev/null @@ -1,71 +0,0 @@ -import dataclasses -import json - -import pytest - -from hats.catalog.catalog_type import CatalogType -from hats.catalog.index.index_catalog_info import IndexCatalogInfo - - -def test_index_catalog_info( - index_catalog_info, index_catalog_info_with_extra, assert_catalog_info_matches_dict -): - info = IndexCatalogInfo(**index_catalog_info) - assert_catalog_info_matches_dict(info, index_catalog_info) - - info = IndexCatalogInfo(**index_catalog_info_with_extra) - assert_catalog_info_matches_dict(info, index_catalog_info) - assert_catalog_info_matches_dict(info, index_catalog_info_with_extra) - - -def test_str(index_catalog_info_with_extra): - correct_string = "" - for name, value in index_catalog_info_with_extra.items(): - correct_string += f" {name} {value}\n" - cat_info = IndexCatalogInfo(**index_catalog_info_with_extra) - assert str(cat_info) == correct_string - - -def test_read_from_file(index_catalog_info_file, assert_catalog_info_matches_dict): - catalog_info = IndexCatalogInfo.read_from_metadata_file(index_catalog_info_file) - for column in [ - "catalog_name", - "catalog_type", - "total_rows", - "primary_catalog", - "indexing_column", - "extra_columns", - ]: - assert column in dataclasses.asdict(catalog_info) - - with open(index_catalog_info_file, "r", encoding="utf-8") as cat_info_file: - catalog_info_json = json.load(cat_info_file) - assert_catalog_info_matches_dict(catalog_info, catalog_info_json) - - -def test_required_fields_missing(index_catalog_info): - required_fields = ["primary_catalog", "indexing_column"] - for required_field in required_fields: - assert required_field in IndexCatalogInfo.required_fields - for field in required_fields: - init_data = index_catalog_info.copy() - init_data[field] = None - with pytest.raises(ValueError, match=field): - IndexCatalogInfo(**init_data) - - -def test_type_missing(index_catalog_info): - init_data = index_catalog_info.copy() - init_data["catalog_type"] = None - catalog_info = IndexCatalogInfo(**init_data) - assert catalog_info.catalog_type == CatalogType.INDEX - - -def test_wrong_type(index_catalog_info, catalog_info_data): - with pytest.raises(TypeError, match="unexpected"): - IndexCatalogInfo(**catalog_info_data) - - with pytest.raises(ValueError, match=f"{CatalogType.INDEX}"): - init_data = index_catalog_info.copy() - init_data["catalog_type"] = CatalogType.OBJECT - IndexCatalogInfo(**init_data) diff --git a/tests/hats/catalog/loaders/test_read_hats.py b/tests/hats/catalog/loaders/test_read_hats.py index 2b4e0ac0..693ccdc9 100644 --- a/tests/hats/catalog/loaders/test_read_hats.py +++ b/tests/hats/catalog/loaders/test_read_hats.py @@ -1,12 +1,9 @@ import pytest -from hats.catalog import CatalogType from hats.loaders import read_hats def test_read_hats_wrong_catalog_type(small_sky_dir): - with pytest.raises(ValueError, match="must have type"): - read_hats(small_sky_dir, catalog_type=CatalogType.ASSOCIATION) with pytest.raises(NotImplementedError, match="load catalog of type"): read_hats(small_sky_dir, catalog_type="unknown") diff --git a/tests/hats/catalog/margin_cache/test_margin_cache_catalog_info.py b/tests/hats/catalog/margin_cache/test_margin_cache_catalog_info.py deleted file mode 100644 index 494cfac5..00000000 --- a/tests/hats/catalog/margin_cache/test_margin_cache_catalog_info.py +++ /dev/null @@ -1,68 +0,0 @@ -import dataclasses -import json - -import pytest - -from hats.catalog.catalog_type import CatalogType -from hats.catalog.margin_cache.margin_cache_catalog_info import MarginCacheCatalogInfo - - -def test_margin_cache_catalog_info(margin_cache_catalog_info_data, assert_catalog_info_matches_dict): - info = MarginCacheCatalogInfo(**margin_cache_catalog_info_data) - assert_catalog_info_matches_dict(info, margin_cache_catalog_info_data) - - -def test_str(margin_cache_catalog_info_data): - correct_string = "" - for name, value in margin_cache_catalog_info_data.items(): - correct_string += f" {name} {value}\n" - cat_info = MarginCacheCatalogInfo(**margin_cache_catalog_info_data) - assert str(cat_info) == correct_string - - -def test_read_from_file(margin_cache_catalog_info_file, assert_catalog_info_matches_dict): - catalog_info = MarginCacheCatalogInfo.read_from_metadata_file(margin_cache_catalog_info_file) - for column in [ - "catalog_name", - "catalog_type", - "total_rows", - "epoch", - "ra_column", - "dec_column", - "primary_catalog", - "margin_threshold", - ]: - assert column in dataclasses.asdict(catalog_info) - - with open(margin_cache_catalog_info_file, "r", encoding="utf-8") as cat_info_file: - catalog_info_json = json.load(cat_info_file) - assert_catalog_info_matches_dict(catalog_info, catalog_info_json) - - -def test_required_fields_missing(margin_cache_catalog_info_data): - required_fields = ["primary_catalog", "margin_threshold"] - for required_field in required_fields: - assert required_field in MarginCacheCatalogInfo.required_fields - for field in required_fields: - init_data = margin_cache_catalog_info_data.copy() - init_data[field] = None - with pytest.raises(ValueError, match=field): - MarginCacheCatalogInfo(**init_data) - - -def test_type_missing(margin_cache_catalog_info_data): - init_data = margin_cache_catalog_info_data.copy() - init_data["catalog_type"] = None - catalog_info = MarginCacheCatalogInfo(**init_data) - assert catalog_info.catalog_type == CatalogType.MARGIN - - -def test_wrong_type(margin_cache_catalog_info_data, catalog_info_data): - with pytest.raises(TypeError, match="unexpected"): - unexpected_info = {**catalog_info_data, "invalid_key": "unknown"} - MarginCacheCatalogInfo(**unexpected_info) - - with pytest.raises(ValueError, match=f"{CatalogType.MARGIN}"): - init_data = margin_cache_catalog_info_data.copy() - init_data["catalog_type"] = CatalogType.OBJECT - MarginCacheCatalogInfo(**init_data) diff --git a/tests/hats/catalog/margin_cache/test_margin_catalog.py b/tests/hats/catalog/margin_cache/test_margin_catalog.py index ccfab571..c3bb4503 100644 --- a/tests/hats/catalog/margin_cache/test_margin_catalog.py +++ b/tests/hats/catalog/margin_cache/test_margin_catalog.py @@ -1,10 +1,10 @@ -import json import os import pyarrow as pa import pytest from hats.catalog import CatalogType, MarginCatalog, PartitionInfo +from hats.catalog.dataset.table_properties import TableProperties from hats.loaders import read_hats from hats.pixel_math import HealpixPixel @@ -21,18 +21,6 @@ def test_init_catalog(margin_catalog_info, margin_catalog_pixels): assert hp_pixel in catalog.pixel_tree -def test_wrong_catalog_type(margin_catalog_info, margin_catalog_pixels): - margin_catalog_info.catalog_type = CatalogType.OBJECT - with pytest.raises(ValueError, match="catalog_type"): - MarginCatalog(margin_catalog_info, margin_catalog_pixels) - - -def test_wrong_catalog_info_type(catalog_info, margin_catalog_pixels): - catalog_info.catalog_type = CatalogType.MARGIN - with pytest.raises(TypeError, match="catalog_info"): - MarginCatalog(catalog_info, margin_catalog_pixels) - - def test_read_from_file(margin_catalog_path, margin_catalog_pixels, margin_catalog_schema): catalog = read_hats(margin_catalog_path) @@ -45,7 +33,6 @@ def test_read_from_file(margin_catalog_path, margin_catalog_pixels, margin_catal info = catalog.catalog_info assert info.catalog_name == "small_sky_order1_margin" assert info.catalog_type == CatalogType.MARGIN - assert info.epoch == "J2000" assert info.ra_column == "ra" assert info.dec_column == "dec" assert info.primary_catalog == "small_sky_order1" @@ -66,13 +53,12 @@ def test_empty_directory(tmp_path, margin_cache_catalog_info_data, margin_catalo os.makedirs(catalog_path, exist_ok=True) ## Path exists but there's nothing there - with pytest.raises(FileNotFoundError, match="catalog_info"): + with pytest.raises(FileNotFoundError, match="properties file"): read_hats(catalog_path) ## catalog_info file exists - getting closer - file_name = catalog_path / "catalog_info.json" - with open(file_name, "w", encoding="utf-8") as metadata_file: - metadata_file.write(json.dumps(margin_cache_catalog_info_data)) + properties = TableProperties(**margin_cache_catalog_info_data) + properties.to_properties_file(catalog_path) with pytest.raises(FileNotFoundError, match="metadata"): read_hats(catalog_path) diff --git a/tests/hats/catalog/source_catalog/test_source_catalog_info.py b/tests/hats/catalog/source_catalog/test_source_catalog_info.py deleted file mode 100644 index 9e905eb6..00000000 --- a/tests/hats/catalog/source_catalog/test_source_catalog_info.py +++ /dev/null @@ -1,71 +0,0 @@ -import dataclasses -import json - -import pytest - -from hats.catalog.catalog_type import CatalogType -from hats.catalog.source_catalog.source_catalog_info import SourceCatalogInfo - - -def test_source_catalog_info( - source_catalog_info, - source_catalog_info_with_extra, - assert_catalog_info_matches_dict, -): - info = SourceCatalogInfo(**source_catalog_info) - assert_catalog_info_matches_dict(info, source_catalog_info) - - info = SourceCatalogInfo(**source_catalog_info_with_extra) - assert_catalog_info_matches_dict(info, source_catalog_info) - assert_catalog_info_matches_dict(info, source_catalog_info_with_extra) - - -def test_str(source_catalog_info_with_extra): - correct_string = "" - for name, value in source_catalog_info_with_extra.items(): - correct_string += f" {name} {value}\n" - cat_info = SourceCatalogInfo(**source_catalog_info_with_extra) - assert str(cat_info) == correct_string - - -def test_read_from_file(source_catalog_info_file, assert_catalog_info_matches_dict): - catalog_info = SourceCatalogInfo.read_from_metadata_file(source_catalog_info_file) - for column in [ - "catalog_name", - "catalog_type", - "total_rows", - "primary_catalog", - ]: - assert column in dataclasses.asdict(catalog_info) - - with open(source_catalog_info_file, "r", encoding="utf-8") as cat_info_file: - catalog_info_json = json.load(cat_info_file) - assert_catalog_info_matches_dict(catalog_info, catalog_info_json) - - -def test_required_fields_missing(source_catalog_info): - required_fields = ["epoch", "ra_column", "dec_column"] - for required_field in required_fields: - assert required_field in SourceCatalogInfo.required_fields - for field in required_fields: - init_data = source_catalog_info.copy() - init_data[field] = None - with pytest.raises(ValueError, match=field): - SourceCatalogInfo(**init_data) - - -def test_type_missing(source_catalog_info): - init_data = source_catalog_info.copy() - init_data["catalog_type"] = None - catalog_info = SourceCatalogInfo(**init_data) - assert catalog_info.catalog_type == CatalogType.SOURCE - - -def test_wrong_type(source_catalog_info, catalog_info_data): - with pytest.raises(ValueError, match=f"{CatalogType.SOURCE}"): - SourceCatalogInfo(**catalog_info_data) - - with pytest.raises(ValueError, match=f"{CatalogType.SOURCE}"): - init_data = source_catalog_info.copy() - init_data["catalog_type"] = CatalogType.OBJECT - SourceCatalogInfo(**init_data) diff --git a/tests/hats/catalog/test_catalog.py b/tests/hats/catalog/test_catalog.py index 26fb46f7..0133a95f 100644 --- a/tests/hats/catalog/test_catalog.py +++ b/tests/hats/catalog/test_catalog.py @@ -10,6 +10,7 @@ import hats.pixel_math.healpix_shim as hp from hats.catalog import Catalog, CatalogType, PartitionInfo +from hats.catalog.dataset.table_properties import TableProperties from hats.io import paths from hats.io.file_io import read_fits_image from hats.loaders import read_hats @@ -28,11 +29,6 @@ def test_catalog_load(catalog_info, catalog_pixels): assert hp_pixel in catalog.pixel_tree -def test_catalog_load_wrong_catalog_info(base_catalog_info, catalog_pixels): - with pytest.raises(TypeError): - Catalog(base_catalog_info, catalog_pixels) - - def test_catalog_wrong_catalog_type(catalog_info, catalog_pixels): catalog_info.catalog_type = CatalogType.INDEX with pytest.raises(ValueError): @@ -476,7 +472,7 @@ def test_box_filter_invalid_args(small_sky_order1_catalog): small_sky_order1_catalog.filter_by_box(ra=None, dec=None) -def test_empty_directory(tmp_path): +def test_empty_directory(tmp_path, catalog_info_data): """Test loading empty or incomplete data""" ## Path doesn't exist with pytest.raises(FileNotFoundError): @@ -486,13 +482,12 @@ def test_empty_directory(tmp_path): os.makedirs(catalog_path, exist_ok=True) ## Path exists but there's nothing there - with pytest.raises(FileNotFoundError, match="catalog_info"): + with pytest.raises(FileNotFoundError, match="properties file"): read_hats(catalog_path) ## catalog_info file exists - getting closer - file_name = catalog_path / "catalog_info.json" - with open(file_name, "w", encoding="utf-8") as metadata_file: - metadata_file.write('{"catalog_name":"empty", "catalog_type":"source"}') + properties = TableProperties(**catalog_info_data) + properties.to_properties_file(catalog_path) with pytest.raises(FileNotFoundError, match="metadata"): read_hats(catalog_path) @@ -503,7 +498,7 @@ def test_empty_directory(tmp_path): with pytest.warns(UserWarning, match="slow"): catalog = read_hats(catalog_path) - assert catalog.catalog_name == "empty" + assert catalog.catalog_name == "test_name" def test_generate_negative_tree_pixels(small_sky_order1_catalog): diff --git a/tests/hats/catalog/test_catalog_info.py b/tests/hats/catalog/test_catalog_info.py deleted file mode 100644 index 4bec678d..00000000 --- a/tests/hats/catalog/test_catalog_info.py +++ /dev/null @@ -1,43 +0,0 @@ -import dataclasses -import json - -from hats.catalog.catalog_info import CatalogInfo - - -def test_catalog_info(catalog_info_data, assert_catalog_info_matches_dict): - info = CatalogInfo(**catalog_info_data) - assert_catalog_info_matches_dict(info, catalog_info_data) - - -def test_catalog_info_defaults(base_catalog_info_data, assert_catalog_info_matches_dict): - info = CatalogInfo(**base_catalog_info_data) - actual_catalog_info = base_catalog_info_data.copy() - actual_catalog_info["epoch"] = "J2000" - actual_catalog_info["ra_column"] = "ra" - actual_catalog_info["dec_column"] = "dec" - assert_catalog_info_matches_dict(info, actual_catalog_info) - - -def test_str(catalog_info_data): - correct_string = "" - for name, value in catalog_info_data.items(): - correct_string += f" {name} {value}\n" - cat_info = CatalogInfo(**catalog_info_data) - assert str(cat_info) == correct_string - - -def test_read_from_file(catalog_info_file, assert_catalog_info_matches_dict): - catalog_info = CatalogInfo.read_from_metadata_file(catalog_info_file) - for column in [ - "catalog_name", - "catalog_type", - "total_rows", - "epoch", - "ra_column", - "dec_column", - ]: - assert column in dataclasses.asdict(catalog_info) - - with open(catalog_info_file, "r", encoding="utf-8") as cat_info_file: - catalog_info_json = json.load(cat_info_file) - assert_catalog_info_matches_dict(catalog_info, catalog_info_json) diff --git a/tests/hats/inspection/test_almanac_info.py b/tests/hats/inspection/test_almanac_info.py index 72677098..604df1e9 100644 --- a/tests/hats/inspection/test_almanac_info.py +++ b/tests/hats/inspection/test_almanac_info.py @@ -66,14 +66,14 @@ def test_write_to_bad_file(tmp_path, small_sky_dir): almanac_info.write_to_file(default_dir=True) -def test_association_fields(association_catalog_path, index_catalog_info_file, small_sky_dir): +def test_association_fields(association_catalog_path, test_data_dir, small_sky_dir): """Test additional text fields tables with primary/join relationships.""" almanac_info = AlmanacInfo.from_catalog_dir(association_catalog_path) assert almanac_info.catalog_name == "small_sky_to_small_sky_order1" assert almanac_info.primary == "small_sky" assert almanac_info.join == "small_sky_order1" - almanac_info = AlmanacInfo.from_catalog_dir(index_catalog_info_file) + almanac_info = AlmanacInfo.from_catalog_dir(test_data_dir / "info_only" / "index_catalog") assert almanac_info.primary == "catalog" assert almanac_info.join is None diff --git a/tests/hats/io/file_io/test_file_io.py b/tests/hats/io/file_io/test_file_io.py index 10a71135..c0f6e14d 100644 --- a/tests/hats/io/file_io/test_file_io.py +++ b/tests/hats/io/file_io/test_file_io.py @@ -1,5 +1,3 @@ -import json - import numpy as np import pandas as pd import pytest @@ -8,7 +6,6 @@ delete_file, load_csv_to_pandas, load_csv_to_pandas_generator, - load_json_file, make_directory, read_parquet_dataset, read_parquet_file_to_pandas, @@ -76,15 +73,6 @@ def test_write_string_to_file(tmp_path): assert not does_file_or_directory_exist(test_file_path) -def test_load_json(small_sky_dir): - catalog_info_path = small_sky_dir / "catalog_info.json" - json_dict = None - with open(catalog_info_path, "r", encoding="utf-8") as json_file: - json_dict = json.load(json_file) - loaded_json_dict = load_json_file(catalog_info_path, encoding="utf-8") - assert loaded_json_dict == json_dict - - def test_load_csv_to_pandas(small_sky_source_dir): partition_info_path = small_sky_source_dir / "partition_info.csv" frame = load_csv_to_pandas(partition_info_path) diff --git a/tests/hats/io/file_io/test_file_pointers.py b/tests/hats/io/file_io/test_file_pointers.py index 6d09493a..0b1f43fc 100644 --- a/tests/hats/io/file_io/test_file_pointers.py +++ b/tests/hats/io/file_io/test_file_pointers.py @@ -13,7 +13,7 @@ def test_file_or_dir_exist(small_sky_dir): assert does_file_or_directory_exist(small_sky_dir) - catalog_info_string = small_sky_dir / "catalog_info.json" + catalog_info_string = small_sky_dir / "properties" assert does_file_or_directory_exist(catalog_info_string) @@ -28,8 +28,8 @@ def test_append_paths_to_pointer(tmp_path): def test_is_regular_file(small_sky_dir): - catalog_info_file = small_sky_dir / "catalog_info.json" - assert is_regular_file(catalog_info_file) + properties_file = small_sky_dir / "properties" + assert is_regular_file(properties_file) assert not is_regular_file(small_sky_dir) @@ -39,11 +39,11 @@ def test_is_regular_file(small_sky_dir): def test_find_files_matching_path(small_sky_dir): ## no_wildcard - matching_files = find_files_matching_path(small_sky_dir, "catalog_info.json") + matching_files = find_files_matching_path(small_sky_dir, "properties") assert len(matching_files) == 1 - ## wilcard in the name (matches catalog_info and provenance_info) - assert len(find_files_matching_path(small_sky_dir, "*.json")) == 2 + ## wilcard in the name (matches _metadata and _common_metadata) + assert len(find_files_matching_path(small_sky_dir, "*metadata*")) == 2 def test_find_files_matching_path_directory(small_sky_order1_dir): @@ -68,10 +68,9 @@ def test_get_directory_contents(small_sky_order1_dir, tmp_path): "README.md", "_common_metadata", "_metadata", - "catalog_info.json", "partition_info.csv", "point_map.fits", - "provenance_info.json", + "properties", ] expected = [small_sky_order1_dir / file_name for file_name in expected] diff --git a/tests/hats/io/test_validation.py b/tests/hats/io/test_validation.py index 0cd698f2..bd66f5d5 100644 --- a/tests/hats/io/test_validation.py +++ b/tests/hats/io/test_validation.py @@ -7,7 +7,6 @@ import pytest from hats.catalog import PartitionInfo -from hats.io import paths, write_catalog_info from hats.io.validation import is_valid_catalog @@ -17,7 +16,7 @@ def test_is_valid_catalog(tmp_path, small_sky_catalog, small_sky_pixels): assert not is_valid_catalog(tmp_path) # Having the catalog_info file is not enough - write_catalog_info(tmp_path, small_sky_catalog.catalog_info) + small_sky_catalog.catalog_info.to_properties_file(tmp_path) assert not is_valid_catalog(tmp_path) # The catalog is valid if both the catalog_info and _metadata files exist, @@ -27,13 +26,12 @@ def test_is_valid_catalog(tmp_path, small_sky_catalog, small_sky_pixels): assert is_valid_catalog(tmp_path) # A partition_info file alone is also not enough - catalog_info_pointer = paths.get_catalog_info_pointer(tmp_path) - os.remove(catalog_info_pointer) + os.remove(tmp_path / "properties") assert not is_valid_catalog(tmp_path) # The catalog_info file needs to be in the correct format small_sky_catalog.catalog_info.catalog_type = "invalid" - write_catalog_info(tmp_path, small_sky_catalog.catalog_info) + small_sky_catalog.catalog_info.to_properties_file(tmp_path) assert not is_valid_catalog(tmp_path) @@ -51,7 +49,7 @@ def test_is_valid_catalog_strict(tmp_path, small_sky_catalog, small_sky_pixels, assert not is_valid_catalog(tmp_path, **flags) # Having the catalog_info file is not enough - write_catalog_info(tmp_path, small_sky_catalog.catalog_info) + small_sky_catalog.catalog_info.to_properties_file(tmp_path) with pytest.warns(): assert not is_valid_catalog(tmp_path, **flags) @@ -90,11 +88,11 @@ def test_is_valid_catalog_fail_fast(tmp_path, small_sky_catalog, small_sky_pixel "verbose": False, # don't bother printing anything. } # An empty directory means an invalid catalog - with pytest.raises(ValueError, match="catalog_info.json"): + with pytest.raises(ValueError, match="properties file"): is_valid_catalog(tmp_path, **flags) # Having the catalog_info file is not enough - write_catalog_info(tmp_path, small_sky_catalog.catalog_info) + small_sky_catalog.catalog_info.to_properties_file(tmp_path) with pytest.raises(ValueError, match="partition_info.csv"): is_valid_catalog(tmp_path, **flags) @@ -128,7 +126,7 @@ def test_is_valid_catalog_verbose_fail(tmp_path, capsys): captured = capsys.readouterr().out assert "Validating catalog at path" in captured - assert "catalog_info.json file does not exist or is invalid" in captured + assert "properties file does not exist or is invalid" in captured assert "partition_info.csv file does not exist" in captured assert "_metadata file does not exist" in captured assert "_common_metadata file does not exist" in captured @@ -155,7 +153,7 @@ def test_valid_catalog_strict_all(small_sky_source_dir, small_sky_order1_dir, sm flags = { "strict": True, # more intensive checks "fail_fast": False, # check everything, and just return true/false - "verbose": False, # don't bother printing anything. + "verbose": True, # don't bother printing anything. } assert is_valid_catalog(small_sky_source_dir, **flags) assert is_valid_catalog(small_sky_order1_dir, **flags) diff --git a/tests/hats/io/test_write_metadata.py b/tests/hats/io/test_write_metadata.py index bf4e9245..8fdb1e07 100644 --- a/tests/hats/io/test_write_metadata.py +++ b/tests/hats/io/test_write_metadata.py @@ -3,7 +3,6 @@ import numpy as np import numpy.testing as npt import pytest -from upath import UPath import hats.io.write_metadata as io import hats.pixel_math as hist @@ -12,125 +11,6 @@ from hats.pixel_math.healpix_pixel import HealpixPixel -def test_write_json_file(assert_text_file_matches, tmp_path): - """Test of arbitrary json dictionary with strings and numbers""" - - expected_lines = [ - "{\n", - ' "first_english": "a",', - ' "first_greek": "alpha",', - ' "first_number": 1,', - r' "first_five_fib": \[', - " 1,", - " 1,", - " 2,", - " 3,", - " 5", - " ]", - r' "integer_type": "",', - ' "np_int": 5000000,', - ' "np_float": 1.618,', - ' "pixel": "Order: 5, Pixel: 9000",', - r' "pixels": \[', - ' "Order: 5, Pixel: 9000",', - ' "Order: 5, Pixel: 9001"', - " ]", - "}", - ] - - dictionary = {} - dictionary["first_english"] = "a" - dictionary["first_greek"] = "alpha" - dictionary["first_number"] = 1 - dictionary["first_five_fib"] = [1, 1, 2, 3, 5] - dictionary["integer_type"] = int - dictionary["np_int"] = np.uint64(5_000_000) - dictionary["np_float"] = np.float64(1.618) - dictionary["pixel"] = HealpixPixel(5, 9_000) - dictionary["pixels"] = np.array([HealpixPixel(5, 9_000), HealpixPixel(5, 9_001)]) - - json_filename = tmp_path / "dictionary.json" - io.write_json_file(dictionary, json_filename) - assert_text_file_matches(expected_lines, json_filename) - - -def test_write_json_paths(assert_text_file_matches, tmp_path): - pathlib_path = UPath(tmp_path) - dictionary = {} - dictionary["pathlib_path"] = pathlib_path - dictionary["file_pointer"] = tmp_path - dictionary["first_greek"] = "alpha" - expected_lines = [ - "{\n", - f' "pathlib_path": "{tmp_path}",', - f' "file_pointer": "{tmp_path}",', - ' "first_greek": "alpha"', - "}", - ] - json_filename = tmp_path / "dictionary.json" - io.write_json_file(dictionary, json_filename) - assert_text_file_matches(expected_lines, json_filename) - - -def test_write_catalog_info(assert_text_file_matches, tmp_path, catalog_info): - """Test that we accurately write out catalog metadata""" - catalog_base_dir = tmp_path / "test_name" - file_io.make_directory(catalog_base_dir) - expected_lines = [ - "{", - ' "catalog_name": "test_name",', - ' "catalog_type": "object",', - ' "total_rows": 10,', - ' "epoch": "J2000",', - ' "ra_column": "ra",', - ' "dec_column": "dec"', - "}", - ] - - io.write_catalog_info(dataset_info=catalog_info, catalog_base_dir=catalog_base_dir) - metadata_filename = catalog_base_dir / "catalog_info.json" - assert_text_file_matches(expected_lines, metadata_filename) - - -def test_write_provenance_info(assert_text_file_matches, tmp_path, catalog_info): - """Test that we accurately write out tool-provided generation metadata""" - catalog_base_dir = tmp_path / "test_name" - file_io.make_directory(catalog_base_dir) - expected_lines = [ - "{", - ' "catalog_name": "test_name",', - ' "catalog_type": "object",', - ' "total_rows": 10,', - ' "epoch": "J2000",', - ' "ra_column": "ra",', - ' "dec_column": "dec",', - r' "version": ".*",', # version matches digits - r' "generation_date": "[.\d]+",', # date matches date format - ' "tool_args": {', - ' "tool_name": "hipscat-import",', - ' "tool_version": "1.0.0",', - r' "input_file_names": \[', - ' "file1",', - ' "file2",', - ' "file3"', - " ]", - " }", - "}", - ] - - tool_args = { - "tool_name": "hipscat-import", - "tool_version": "1.0.0", - "input_file_names": ["file1", "file2", "file3"], - } - - io.write_provenance_info( - catalog_base_dir=catalog_base_dir, dataset_info=catalog_info, tool_args=tool_args - ) - metadata_filename = catalog_base_dir / "provenance_info.json" - assert_text_file_matches(expected_lines, metadata_filename) - - def test_write_partition_info_healpix_pixel_map(assert_text_file_matches, tmp_path): """Test that we accurately write out the partition stats for overloaded input""" catalog_base_dir = tmp_path / "test_name" From ae059995e7d683b38d2e9b0a29635001b42bef6b Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Wed, 25 Sep 2024 13:50:34 -0400 Subject: [PATCH 4/7] Support hips-style space-delimited lists --- src/hats/catalog/dataset/table_properties.py | 20 +++++++++++++++++-- tests/data/info_only/catalog/properties | 15 +++++++------- tests/data/info_only/dataset/properties | 15 +++++++------- tests/data/info_only/index_catalog/properties | 14 +++++++------ tests/data/info_only/margin_cache/properties | 17 ++++++++-------- .../catalog/dataset/test_table_properties.py | 9 ++++++--- 6 files changed, 57 insertions(+), 33 deletions(-) diff --git a/src/hats/catalog/dataset/table_properties.py b/src/hats/catalog/dataset/table_properties.py index 4949d86b..6c408e13 100644 --- a/src/hats/catalog/dataset/table_properties.py +++ b/src/hats/catalog/dataset/table_properties.py @@ -1,7 +1,7 @@ from typing import List, Optional from jproperties import Properties -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator, model_validator from typing_extensions import Self from upath import UPath @@ -84,6 +84,22 @@ class TableProperties(BaseModel): ## Allow any extra keyword args to be stored on the properties object. model_config = ConfigDict(extra="allow", populate_by_name=True, use_enum_values=True) + @field_validator("extra_columns", mode="before") + @classmethod + def space_delimited_list(cls, str_value: str) -> List[str]: + """Convert a space-delimited list string into a python list of strings.""" + if isinstance(str_value, str): + if str_value: + str_value = str_value.strip() + if str_value: + return str_value.split(" ") + return str_value + + @field_serializer("extra_columns") + def serialize_space_delimited_list(self, str_list) -> str: + """Convert a python list of strings into a space-delimited string.""" + return " ".join(str_list) + @model_validator(mode="after") def check_allowed_and_required(self) -> Self: """Check that type-specific fields are appropriate, and required fields are set.""" @@ -143,4 +159,4 @@ def to_properties_file(self, catalog_dir: UPath) -> Self: properties._key_order = parameters.keys() file_path = file_io.get_upath(catalog_dir) / "properties" with file_path.open("wb") as _file: - properties.store(_file, "utf-8", timestamp=False) + properties.store(_file, encoding="utf-8", initial_comments="HATS catalog", timestamp=False) diff --git a/tests/data/info_only/catalog/properties b/tests/data/info_only/catalog/properties index 83fef746..1701ec63 100644 --- a/tests/data/info_only/catalog/properties +++ b/tests/data/info_only/catalog/properties @@ -1,7 +1,8 @@ -obs_collection = catalog -dataproduct_type = object -hats_col_j2000_ra = ra -hats_col_j2000_dec = dec -hats_max_rows = 1000 -hats_nrows = 10 -hats_order = 0 \ No newline at end of file +#HATS catalog +obs_collection=catalog +dataproduct_type=object +hats_nrows=10 +hats_col_j2000_ra=ra +hats_col_j2000_dec=dec +hats_max_rows=1000 +hats_order=0 diff --git a/tests/data/info_only/dataset/properties b/tests/data/info_only/dataset/properties index e7808ef3..b21251e7 100644 --- a/tests/data/info_only/dataset/properties +++ b/tests/data/info_only/dataset/properties @@ -1,7 +1,8 @@ -obs_collection = dataset -dataproduct_type = object -hats_col_j2000_ra = ra -hats_col_j2000_dec = dec -hats_max_rows = 1000 -hats_nrows = 10 -hats_order = 0 \ No newline at end of file +#HATS catalog +obs_collection=dataset +dataproduct_type=object +hats_nrows=10 +hats_col_j2000_ra=ra +hats_col_j2000_dec=dec +hats_max_rows=1000 +hats_order=0 diff --git a/tests/data/info_only/index_catalog/properties b/tests/data/info_only/index_catalog/properties index 5a62e29a..e1a32413 100644 --- a/tests/data/info_only/index_catalog/properties +++ b/tests/data/info_only/index_catalog/properties @@ -1,7 +1,9 @@ -obs_collection = index_catalog -dataproduct_type = index -hats_max_rows = 1000 -hats_nrows = 100 -hats_order = 0 +#HATS catalog +obs_collection=index_catalog +dataproduct_type=index +hats_nrows=100 +hats_primary_table_url=catalog hats_index_column=id -hats_primary_table_url=catalog \ No newline at end of file +hats_index_extra_column=mjd band +hats_max_rows=1000 +hats_order=0 diff --git a/tests/data/info_only/margin_cache/properties b/tests/data/info_only/margin_cache/properties index 7c0b5177..80a6acc9 100644 --- a/tests/data/info_only/margin_cache/properties +++ b/tests/data/info_only/margin_cache/properties @@ -1,9 +1,10 @@ -obs_collection = margin_cache -dataproduct_type = margin -hats_col_j2000_ra = ra -hats_col_j2000_dec = dec -hats_max_rows = 1000 -hats_nrows = 100 -hats_order = 0 +#HATS catalog +obs_collection=margin_cache +dataproduct_type=margin +hats_nrows=100 +hats_col_j2000_ra=ra +hats_col_j2000_dec=dec +hats_primary_table_url=catalog hats_margin_threshold=0.5 -hats_primary_table_url=catalog \ No newline at end of file +hats_max_rows=1000 +hats_order=0 diff --git a/tests/hats/catalog/dataset/test_table_properties.py b/tests/hats/catalog/dataset/test_table_properties.py index 1c9bb407..0b6ebb07 100644 --- a/tests/hats/catalog/dataset/test_table_properties.py +++ b/tests/hats/catalog/dataset/test_table_properties.py @@ -1,11 +1,14 @@ +import pytest + from hats.catalog.dataset.table_properties import TableProperties -def test_read_from_file_round_trip(dataset_path, tmp_path): - table_properties = TableProperties.read_from_dir(dataset_path) +@pytest.mark.parametrize("data_dir", ["catalog", "dataset", "index_catalog", "margin_cache"]) +def test_read_from_file_round_trip(test_data_dir, data_dir, tmp_path): + dataset_path = test_data_dir / "info_only" / data_dir + table_properties = TableProperties.read_from_dir(dataset_path) table_properties.to_properties_file(tmp_path) - round_trip_properties = TableProperties.read_from_dir(tmp_path) assert table_properties == round_trip_properties From 9d0e97a55591e3d22cf1e11c3a0bd96787442dbd Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Wed, 25 Sep 2024 14:21:09 -0400 Subject: [PATCH 5/7] Good type hint --- src/hats/catalog/dataset/table_properties.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/hats/catalog/dataset/table_properties.py b/src/hats/catalog/dataset/table_properties.py index 6c408e13..5f36bc3d 100644 --- a/src/hats/catalog/dataset/table_properties.py +++ b/src/hats/catalog/dataset/table_properties.py @@ -1,4 +1,5 @@ -from typing import List, Optional +from pathlib import Path +from typing import List, Optional, Union from jproperties import Properties from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator, model_validator @@ -140,7 +141,7 @@ def __str__(self): return formatted_string @classmethod - def read_from_dir(cls, catalog_dir: UPath) -> Self: + def read_from_dir(cls, catalog_dir: Union[str, Path, UPath]) -> Self: """Read field values from a java-style properties file.""" file_path = file_io.get_upath(catalog_dir) / "properties" if not file_io.does_file_or_directory_exist(file_path): @@ -150,7 +151,7 @@ def read_from_dir(cls, catalog_dir: UPath) -> Self: p.load(f, "utf-8") return cls(**p.properties) - def to_properties_file(self, catalog_dir: UPath) -> Self: + def to_properties_file(self, catalog_dir: Union[str, Path, UPath]) -> Self: """Write fields to a java-style properties file.""" # pylint: disable=protected-access parameters = self.model_dump(by_alias=True, exclude_none=True) From aa9985eaa166e65fb6a408570ae8000903331092 Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Wed, 25 Sep 2024 14:56:55 -0400 Subject: [PATCH 6/7] Reduce duplication. --- .../partition_join_info.py | 4 +- src/hats/catalog/catalog.py | 42 +------------------ src/hats/catalog/partition_info.py | 2 +- src/hats/inspection/almanac.py | 2 +- tests/hats/catalog/test_catalog.py | 9 +--- 5 files changed, 6 insertions(+), 53 deletions(-) diff --git a/src/hats/catalog/association_catalog/partition_join_info.py b/src/hats/catalog/association_catalog/partition_join_info.py index 8f5b2107..58ae6784 100644 --- a/src/hats/catalog/association_catalog/partition_join_info.py +++ b/src/hats/catalog/association_catalog/partition_join_info.py @@ -73,7 +73,7 @@ def primary_to_join_map(self) -> Dict[HealpixPixel, List[HealpixPixel]]: return primary_to_join def write_to_metadata_files(self, catalog_path: str | Path | UPath | None = None): - """Generate parquet metadata, using the known partitions. + """Generate parquet metadata, using the known joint partitions. Args: catalog_path (UPath): base path for the catalog @@ -86,7 +86,7 @@ def write_to_metadata_files(self, catalog_path: str | Path | UPath | None = None """ if catalog_path is None: if self.catalog_base_dir is None: - raise ValueError("catalog_path is required if info was not loaded from a directory") + raise ValueError("catalog_path is required if join info was not loaded from a directory") catalog_path = self.catalog_base_dir batches = [ diff --git a/src/hats/catalog/catalog.py b/src/hats/catalog/catalog.py index 86bfe9f5..9b694ce4 100644 --- a/src/hats/catalog/catalog.py +++ b/src/hats/catalog/catalog.py @@ -2,18 +2,12 @@ from __future__ import annotations -from pathlib import Path from typing import List, Tuple import numpy as np -import pyarrow as pa -from mocpy import MOC -from upath import UPath import hats.pixel_math.healpix_shim as hp -from hats.catalog.catalog_type import CatalogType -from hats.catalog.dataset.table_properties import TableProperties -from hats.catalog.healpix_dataset.healpix_dataset import HealpixDataset, PixelInputTypes +from hats.catalog.healpix_dataset.healpix_dataset import HealpixDataset from hats.pixel_math import HealpixPixel from hats.pixel_math.box_filter import generate_box_moc, wrap_ra_angles from hats.pixel_math.cone_filter import generate_cone_moc @@ -35,40 +29,6 @@ class Catalog(HealpixDataset): `Norder=/Dir=/Npix=.parquet` """ - HIPS_CATALOG_TYPES = [CatalogType.OBJECT, CatalogType.SOURCE] - - def __init__( - self, - catalog_info: TableProperties, - pixels: PixelInputTypes, - catalog_path: str | Path | UPath | None = None, - moc: MOC | None = None, - schema: pa.Schema | None = None, - ) -> None: - """Initializes a Catalog - - Args: - catalog_info: TableProperties object with catalog metadata - pixels: Specifies the pixels contained in the catalog. Can be either a - list of HealpixPixel, `PartitionInfo object`, or a `PixelTree` object - catalog_path: If the catalog is stored on disk, specify the location of the catalog - Does not load the catalog from this path, only store as metadata - moc (mocpy.MOC): MOC object representing the coverage of the catalog - schema (pa.Schema): The pyarrow schema for the catalog - """ - if catalog_info.catalog_type not in self.HIPS_CATALOG_TYPES: - raise ValueError( - f"Catalog info `catalog_type` must be one of " - f"{', '.join([t.value for t in self.HIPS_CATALOG_TYPES])}" - ) - super().__init__( - catalog_info, - pixels, - catalog_path=catalog_path, - moc=moc, - schema=schema, - ) - def filter_by_cone(self, ra: float, dec: float, radius_arcsec: float) -> Catalog: """Filter the pixels in the catalog to only include the pixels that overlap with a cone diff --git a/src/hats/catalog/partition_info.py b/src/hats/catalog/partition_info.py index cb81f940..7b936567 100644 --- a/src/hats/catalog/partition_info.py +++ b/src/hats/catalog/partition_info.py @@ -92,7 +92,7 @@ def write_to_metadata_files(self, catalog_path: str | Path | UPath | None = None """ if catalog_path is None: if self.catalog_base_dir is None: - raise ValueError("catalog_path is required if info was not loaded from a directory") + raise ValueError("catalog_path is required if partition info was not loaded from a directory") catalog_path = self.catalog_base_dir batches = [ diff --git a/src/hats/inspection/almanac.py b/src/hats/inspection/almanac.py index 6104e4ea..3d407e1a 100644 --- a/src/hats/inspection/almanac.py +++ b/src/hats/inspection/almanac.py @@ -6,7 +6,7 @@ import pandas as pd -from hats.catalog.catalog import CatalogType +from hats.catalog.catalog_type import CatalogType from hats.catalog.dataset.dataset import Dataset from hats.inspection.almanac_info import AlmanacInfo from hats.io.file_io import file_pointer diff --git a/tests/hats/catalog/test_catalog.py b/tests/hats/catalog/test_catalog.py index 0133a95f..73d28413 100644 --- a/tests/hats/catalog/test_catalog.py +++ b/tests/hats/catalog/test_catalog.py @@ -9,8 +9,7 @@ from mocpy import MOC import hats.pixel_math.healpix_shim as hp -from hats.catalog import Catalog, CatalogType, PartitionInfo -from hats.catalog.dataset.table_properties import TableProperties +from hats.catalog import Catalog, PartitionInfo, TableProperties from hats.io import paths from hats.io.file_io import read_fits_image from hats.loaders import read_hats @@ -29,12 +28,6 @@ def test_catalog_load(catalog_info, catalog_pixels): assert hp_pixel in catalog.pixel_tree -def test_catalog_wrong_catalog_type(catalog_info, catalog_pixels): - catalog_info.catalog_type = CatalogType.INDEX - with pytest.raises(ValueError): - Catalog(catalog_info, catalog_pixels) - - def test_partition_info_pixel_input_types(catalog_info, catalog_pixels): partition_info = PartitionInfo.from_healpix(catalog_pixels) catalog = Catalog(catalog_info, partition_info) From 91d04357fad4ddfd94b77147908a3f9b7865d198 Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Thu, 26 Sep 2024 10:18:16 -0400 Subject: [PATCH 7/7] Changes from code review --- .../association_catalog.py | 3 - src/hats/catalog/dataset/table_properties.py | 38 ++++++------ src/hats/catalog/index/__init__.py | 0 src/hats/catalog/margin_cache/__init__.py | 0 .../catalog/dataset/test_table_properties.py | 60 +++++++++++++++++++ tests/hats/io/test_validation.py | 2 +- 6 files changed, 79 insertions(+), 24 deletions(-) create mode 100644 src/hats/catalog/index/__init__.py create mode 100644 src/hats/catalog/margin_cache/__init__.py diff --git a/src/hats/catalog/association_catalog/association_catalog.py b/src/hats/catalog/association_catalog/association_catalog.py index 7308eab4..cb17824f 100644 --- a/src/hats/catalog/association_catalog/association_catalog.py +++ b/src/hats/catalog/association_catalog/association_catalog.py @@ -9,7 +9,6 @@ from upath import UPath from hats.catalog.association_catalog.partition_join_info import PartitionJoinInfo -from hats.catalog.catalog_type import CatalogType from hats.catalog.dataset.table_properties import TableProperties from hats.catalog.healpix_dataset.healpix_dataset import HealpixDataset, PixelInputTypes from hats.io import file_io, paths @@ -34,8 +33,6 @@ def __init__( moc: MOC | None = None, schema: pa.Schema | None = None, ) -> None: - if not catalog_info.catalog_type == CatalogType.ASSOCIATION: - raise ValueError("Catalog info `catalog_type` must be 'association'") super().__init__(catalog_info, pixels, catalog_path, moc=moc, schema=schema) self.join_info = self._get_partition_join_info_from_pixels(join_pixels) diff --git a/src/hats/catalog/dataset/table_properties.py b/src/hats/catalog/dataset/table_properties.py index 5f36bc3d..1ae322a1 100644 --- a/src/hats/catalog/dataset/table_properties.py +++ b/src/hats/catalog/dataset/table_properties.py @@ -1,5 +1,6 @@ +import re from pathlib import Path -from typing import List, Optional, Union +from typing import Iterable, List, Optional, Union from jproperties import Properties from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator, model_validator @@ -53,34 +54,34 @@ class TableProperties(BaseModel): dec_column: Optional[str] = Field(default=None, alias="hats_col_j2000_dec") primary_catalog: Optional[str] = Field(default=None, alias="hats_primary_table_url") - """Reference to object catalog. Relevant for nested, margin, association, and index""" + """Reference to object catalog. Relevant for nested, margin, association, and index.""" margin_threshold: Optional[float] = Field(default=None, alias="hats_margin_threshold") """Threshold of the pixel boundary, expressed in arcseconds.""" primary_column: Optional[str] = Field(default=None, alias="hats_col_assn_primary") - """Column name in the primary (left) side of join""" + """Column name in the primary (left) side of join.""" primary_column_association: Optional[str] = Field(default=None, alias="hats_col_assn_primary_assn") - """Column name in the association table that matches the primary (left) side of join""" + """Column name in the association table that matches the primary (left) side of join.""" join_catalog: Optional[str] = Field(default=None, alias="hats_assn_join_table_url") - """Catalog name for the joining (right) side of association""" + """Catalog name for the joining (right) side of association.""" join_column: Optional[str] = Field(default=None, alias="hats_col_assn_join") - """Column name in the joining (right) side of join""" + """Column name in the joining (right) side of join.""" join_column_association: Optional[str] = Field(default=None, alias="hats_col_assn_join_assn") - """Column name in the association table that matches the joining (right) side of join""" + """Column name in the association table that matches the joining (right) side of join.""" contains_leaf_files: Optional[bool] = Field(default=None, alias="hats_assn_leaf_files") - """Whether or not the association catalog contains leaf parquet files""" + """Whether or not the association catalog contains leaf parquet files.""" indexing_column: Optional[str] = Field(default=None, alias="hats_index_column") - """Column that we provide an index over""" + """Column that we provide an index over.""" extra_columns: Optional[List[str]] = Field(default=None, alias="hats_index_extra_column") - """Any additional payload columns included in index""" + """Any additional payload columns included in index.""" ## Allow any extra keyword args to be stored on the properties object. model_config = ConfigDict(extra="allow", populate_by_name=True, use_enum_values=True) @@ -90,14 +91,12 @@ class TableProperties(BaseModel): def space_delimited_list(cls, str_value: str) -> List[str]: """Convert a space-delimited list string into a python list of strings.""" if isinstance(str_value, str): - if str_value: - str_value = str_value.strip() - if str_value: - return str_value.split(" ") + # Split on a few kinds of delimiters (just to be safe), and remove duplicates + return list(filter(None, re.split(";| |,|\n", str_value))) return str_value @field_serializer("extra_columns") - def serialize_space_delimited_list(self, str_list) -> str: + def serialize_as_space_delimited_list(self, str_list: Iterable[str]) -> str: """Convert a python list of strings into a space-delimited string.""" return " ".join(str_list) @@ -113,7 +112,7 @@ def check_allowed_and_required(self) -> Self: ) non_allowed = explicit_keys - allowed_keys if len(non_allowed) > 0: - raise ValueError(f"Unexpected property for table type {self.catalog_type} ({non_allowed})") + raise ValueError(f"Unexpected property for table type {self.catalog_type}: {non_allowed}") required_keys = set( CATALOG_TYPE_REQUIRED_FIELDS[self.catalog_type] + ["catalog_name", "catalog_type", "total_rows"] @@ -121,16 +120,15 @@ def check_allowed_and_required(self) -> Self: missing_required = required_keys - explicit_keys if len(missing_required) > 0: raise ValueError( - f"Missing required property for table type {self.catalog_type} ({missing_required})" + f"Missing required property for table type {self.catalog_type}: {missing_required}" ) return self def explicit_dict(self): """Create a dict, based on fields that have been explicitly set, and are not "extra" keys.""" - explicit = self.model_dump(by_alias=False, exclude_none=True, round_trip=True) + explicit = self.model_dump(by_alias=False, exclude_none=True) extra_keys = self.__pydantic_extra__.keys() - explicit = {(key, val) for key, val in explicit.items() if key not in extra_keys} - return dict(explicit) + return {key: val for key, val in explicit.items() if key not in extra_keys} def __str__(self): """Friendly string representation based on named fields.""" diff --git a/src/hats/catalog/index/__init__.py b/src/hats/catalog/index/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/hats/catalog/margin_cache/__init__.py b/src/hats/catalog/margin_cache/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/hats/catalog/dataset/test_table_properties.py b/tests/hats/catalog/dataset/test_table_properties.py index 0b6ebb07..eb3a0e3d 100644 --- a/tests/hats/catalog/dataset/test_table_properties.py +++ b/tests/hats/catalog/dataset/test_table_properties.py @@ -13,6 +13,66 @@ def test_read_from_file_round_trip(test_data_dir, data_dir, tmp_path): assert table_properties == round_trip_properties + kwarg_properties = TableProperties(**round_trip_properties.model_dump(by_alias=False, exclude_none=True)) + assert table_properties == kwarg_properties + + +def test_properties_parsing(): + table_properties = TableProperties( + catalog_name="foo", + catalog_type="index", + total_rows=15, + extra_columns="a , b", + indexing_column="a", + primary_catalog="bar", + unexpected_kwarg="how did this get here", + ) + assert table_properties.extra_columns == ["a", "b"] + + # unexpected_kwarg is not part of the named args, so it shouldn't show up in the debug string + assert ( + str(table_properties) + == """ catalog_name foo + catalog_type index + total_rows 15 + primary_catalog bar + indexing_column a + extra_columns a b +""" + ) + table_properties_using_list = TableProperties( + catalog_name="foo", + catalog_type="index", + total_rows=15, + extra_columns=["a", "b"], + indexing_column="a", + primary_catalog="bar", + unexpected_kwarg="how did this get here", + ) + assert table_properties_using_list == table_properties + + +def test_properties_allowed_required(): + # Missing required field indexing_column + with pytest.raises(ValueError, match="indexing_column"): + TableProperties( + catalog_name="foo", + catalog_type="index", + total_rows=15, + primary_catalog="bar", + ) + + # join_column is only allowed on association catalogs + with pytest.raises(ValueError, match="join_column"): + TableProperties( + catalog_name="foo", + catalog_type="index", + total_rows=15, + indexing_column="a", + primary_catalog="bar", + join_column="b", + ) + def test_read_from_dir_branches( small_sky_dir, diff --git a/tests/hats/io/test_validation.py b/tests/hats/io/test_validation.py index bd66f5d5..23461af1 100644 --- a/tests/hats/io/test_validation.py +++ b/tests/hats/io/test_validation.py @@ -153,7 +153,7 @@ def test_valid_catalog_strict_all(small_sky_source_dir, small_sky_order1_dir, sm flags = { "strict": True, # more intensive checks "fail_fast": False, # check everything, and just return true/false - "verbose": True, # don't bother printing anything. + "verbose": False, # don't bother printing anything. } assert is_valid_catalog(small_sky_source_dir, **flags) assert is_valid_catalog(small_sky_order1_dir, **flags)