Skip to content

Commit

Permalink
minor bugfixes and feature additions
Browse files Browse the repository at this point in the history 
* fixes #55,#54,#53
  • Loading branch information
@michaelaye
michaelaye committed Dec 18, 2023
1 parent 9ad7c58 commit 28b6db8
Show file tree
Hide file tree
Showing 16 changed files with 5,013 additions and 1,910 deletions.
2 changes: 1 addition & 1 deletion notebooks/api/01_utils.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -502,7 +502,7 @@
"\n",
" Useful for checking if there's an updated file available.\n",
" \"\"\"\n",
" with urlopen(url, timeout=10) as conn:\n",
" with urlopen(str(url), timeout=10) as conn:\n",
" t = parse_http_date(conn.headers[\"last-modified\"])\n",
" return t\n",
"\n",
Expand Down
110 changes: 20 additions & 90 deletions notebooks/api/02a_pds.indexes.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,11 @@
},
{
"cell_type": "markdown",
"id": "suited-swedish",
"id": "7c6c9d4c-4dcc-47df-8fc5-0cb3ae8c20bc",
"metadata": {},
"source": [
"# PDS Indexes\n",
"\n",
"> Support tools to work with PDS index files. "
"> Support tools to work with PDS index files."
]
},
{
Expand Down Expand Up @@ -181,7 +180,7 @@
" @property\n",
" def label_filename(self):\n",
" if self.url:\n",
" return Path(self.url.split(\"/\")[-1])\n",
" return Path(str(self.url).split(\"/\")[-1])\n",
" else:\n",
" return Path.ls(self.local_dir, file_exts=[\".lbl\", \".LBL\"])[0]\n",
"\n",
Expand Down Expand Up @@ -275,7 +274,7 @@
" subs = tmp.split(\".\")\n",
" if subs[3] != \"indexes\":\n",
" subs.insert(3, \"indexes\")\n",
" return \".\".join(subs)\n"
" return \".\".join(subs)"
]
},
{
Expand Down Expand Up @@ -333,7 +332,8 @@
"metadata": {},
"outputs": [],
"source": [
"key = \"missions.cassini.iss.indexes.moon_summary\""
"key = \"missions.cassini.iss.indexes.moon_summary\"\n",
"key = \"missions.lro.lroc.edr\""
]
},
{
Expand All @@ -345,33 +345,9 @@
{
"data": {
"text/plain": [
"Key: missions.cassini.iss.indexes.moon_summary\n",
"URL: https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_moon_summary.lbl\n",
"Timestamp: 2019-06-08 16:28:22"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"index = Index(key, check_update=False)\n",
"index"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "88c67a63-9101-4730-8fd9-1eb6d9cea224",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Key: missions.cassini.iss.indexes.moon_summary\n",
"URL: https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_moon_summary.lbl\n",
"Timestamp: 2019-06-08 16:28:22"
"Key: missions.lro.lroc.indexes.edr\n",
"URL: http://pds.lroc.asu.edu/data/LRO-L-LROC-2-EDR-V1.0/LROLRC_0048C/INDEX/CUMINDEX.LBL\n",
"Timestamp: 2023-10-10 15:30:56.195462"
]
},
"execution_count": null,
Expand All @@ -380,6 +356,7 @@
}
],
"source": [
"index = Index(key, check_update=True)\n",
"index"
]
},
Expand All @@ -392,7 +369,7 @@
{
"data": {
"text/plain": [
"'missions.cassini.iss.indexes.moon_summary'"
"'missions.lro.lroc.indexes.edr'"
]
},
"execution_count": null,
Expand All @@ -413,7 +390,7 @@
{
"data": {
"text/plain": [
"Path('COISS_2999_moon_summary.lbl')"
"Path('CUMINDEX.LBL')"
]
},
"execution_count": null,
Expand All @@ -434,7 +411,7 @@
{
"data": {
"text/plain": [
"'https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_moon_summary.lbl'"
"'http://pds.lroc.asu.edu/data/LRO-L-LROC-2-EDR-V1.0/LROLRC_0048C/INDEX/CUMINDEX.LBL'"
]
},
"execution_count": null,
Expand All @@ -455,7 +432,7 @@
{
"data": {
"text/plain": [
"Path('/home/ayek72/mnt/slowdata/planetarypy/missions/cassini/iss/indexes/moon_summary')"
"Path('/home/ayek72/mnt/slowdata/planetarypy/missions/lro/lroc/indexes/edr')"
]
},
"execution_count": null,
Expand Down Expand Up @@ -543,7 +520,7 @@
{
"data": {
"text/plain": [
"'2019-06-08T16:28:22'"
"'2023-10-10T15:30:56.195462'"
]
},
"execution_count": null,
Expand All @@ -564,7 +541,7 @@
{
"data": {
"text/plain": [
"'cassini'"
"'lro'"
]
},
"execution_count": null,
Expand All @@ -585,7 +562,7 @@
{
"data": {
"text/plain": [
"Path('/home/ayek72/mnt/slowdata/planetarypy/missions/cassini/iss/indexes/moon_summary')"
"Path('/home/ayek72/mnt/slowdata/planetarypy/missions/lro/lroc/indexes/edr')"
]
},
"execution_count": null,
Expand All @@ -597,53 +574,6 @@
"index.local_dir"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "974f6ca8-3acd-4b9e-bd70-7a8efd88483f",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "e1ab5de7a540493ebeb922ff4aac20d6",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"COISS_2999_moon_summary.lbl: 0%| | 0/4316 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "c60f0a98999b4d3f97f509f06bf4bd60",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"COISS_2999_moon_summary.tab: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Downloaded /home/ayek72/mnt/slowdata/planetarypy/missions/cassini/iss/indexes/moon_summary/COISS_2999_moon_summary.lbl and /home/ayek72/mnt/slowdata/planetarypy/missions/cassini/iss/indexes/moon_summary/COISS_2999_moon_summary.tab\n",
"Done.\n"
]
}
],
"source": [
"index.download()"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand All @@ -653,7 +583,7 @@
{
"data": {
"text/plain": [
"['missions', 'cassini']"
"['missions', 'lro']"
]
},
"execution_count": null,
Expand All @@ -674,7 +604,7 @@
{
"data": {
"text/plain": [
"'cassini.iss'"
"'lro.lroc'"
]
},
"execution_count": null,
Expand All @@ -695,7 +625,7 @@
{
"data": {
"text/plain": [
"{'indexes': {'index': {'url': 'https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_index.lbl', 'timestamp': '2021-02-21T15:55:27'}, 'inventory': {'url': 'https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_inventory.lbl'}, 'moon_summary': {'url': 'https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_moon_summary.lbl', 'timestamp': '2019-06-08T16:28:22'}, 'ring_summary': {'url': 'https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_ring_summary.lbl', 'timestamp': '2022-08-08T11:54:51.140376'}, 'saturn_summary': {'url': 'https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_saturn_summary.lbl', 'timestamp': '2022-12-21T18:10:29.248055'}}}"
"{'indexes': {'edr': {'url': 'http://pds.lroc.asu.edu/data/LRO-L-LROC-2-EDR-V1.0/LROLRC_0048C/INDEX/CUMINDEX.LBL', 'timestamp': '2023-10-10T15:30:56.195462'}}}"
]
},
"execution_count": null,
Expand Down
59 changes: 28 additions & 31 deletions notebooks/api/02b_pds.ctx_index.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,11 @@
},
{
"cell_type": "markdown",
"id": "immediate-honduras",
"id": "f6af10dd-027a-4f74-94b6-a9ef7c258101",
"metadata": {},
"source": [
"# CTX Index\n",
"\n",
"> Scraping the latest CTX index from the latest added volume.\n",
"\n",
"The CTX Index is special as it's latest URL is dynamic, so it needs to be web-scraped."
"> Scraping the latest CTX index file from the latest added volume."
]
},
{
Expand Down Expand Up @@ -81,7 +78,7 @@
"\n",
" @property\n",
" def latest_index_label_url(self):\n",
" return URL(self.url) / f\"{self.latest_release_folder}/index/cumindex.lbl\"\n"
" return URL(self.url) / f\"{self.latest_release_folder}/index/cumindex.lbl\""
]
},
{
Expand Down Expand Up @@ -157,33 +154,33 @@
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9309</th>\n",
" <td>mrox_4654_md5.txt</td>\n",
" <td>24-Aug-2023 06:11</td>\n",
" <th>9515</th>\n",
" <td>mrox_4757_md5.txt</td>\n",
" <td>15-Nov-2023 23:23</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9310</th>\n",
" <td>mrox_4655/</td>\n",
" <td>24-Aug-2023 05:25</td>\n",
" <th>9516</th>\n",
" <td>mrox_4758/</td>\n",
" <td>15-Nov-2023 23:08</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9311</th>\n",
" <td>mrox_4655_md5.txt</td>\n",
" <td>24-Aug-2023 06:11</td>\n",
" <th>9517</th>\n",
" <td>mrox_4758_md5.txt</td>\n",
" <td>15-Nov-2023 23:23</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9312</th>\n",
" <td>mrox_4656/</td>\n",
" <td>24-Aug-2023 05:27</td>\n",
" <th>9518</th>\n",
" <td>mrox_4759/</td>\n",
" <td>15-Nov-2023 23:10</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9313</th>\n",
" <td>mrox_4656_md5.txt</td>\n",
" <td>24-Aug-2023 06:11</td>\n",
" <th>9519</th>\n",
" <td>mrox_4759_md5.txt</td>\n",
" <td>15-Nov-2023 23:23</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>9312 rows × 2 columns</p>\n",
"<p>9518 rows × 2 columns</p>\n",
"</div>"
],
"text/plain": [
Expand All @@ -194,13 +191,13 @@
"5 mrox_0002_md5.txt 09-Dec-2019 19:14\n",
"6 mrox_0003/ 05-Jun-2007 10:15\n",
"... ... ...\n",
"9309 mrox_4654_md5.txt 24-Aug-2023 06:11\n",
"9310 mrox_4655/ 24-Aug-2023 05:25\n",
"9311 mrox_4655_md5.txt 24-Aug-2023 06:11\n",
"9312 mrox_4656/ 24-Aug-2023 05:27\n",
"9313 mrox_4656_md5.txt 24-Aug-2023 06:11\n",
"9515 mrox_4757_md5.txt 15-Nov-2023 23:23\n",
"9516 mrox_4758/ 15-Nov-2023 23:08\n",
"9517 mrox_4758_md5.txt 15-Nov-2023 23:23\n",
"9518 mrox_4759/ 15-Nov-2023 23:10\n",
"9519 mrox_4759_md5.txt 15-Nov-2023 23:23\n",
"\n",
"[9312 rows x 2 columns]"
"[9518 rows x 2 columns]"
]
},
"execution_count": null,
Expand All @@ -221,7 +218,7 @@
{
"data": {
"text/plain": [
"'mrox_4656/'"
"'mrox_4759/'"
]
},
"execution_count": null,
Expand All @@ -242,7 +239,7 @@
{
"data": {
"text/plain": [
"'4656'"
"'4759'"
]
},
"execution_count": null,
Expand All @@ -263,7 +260,7 @@
{
"data": {
"text/plain": [
"URL('https://planetarydata.jpl.nasa.gov/img/data/mro/mars_reconnaissance_orbiter/ctx/mrox_4656/index/cumindex.lbl')"
"URL('https://planetarydata.jpl.nasa.gov/img/data/mro/mars_reconnaissance_orbiter/ctx/mrox_4759/index/cumindex.lbl')"
]
},
"execution_count": null,
Expand Down
Loading

0 comments on commit 28b6db8

Please sign in to comment.