Skip to content

Commit

Permalink
Adds few more archive/cda MMS products as example
Browse files Browse the repository at this point in the history 
Signed-off-by: Alexis Jeandet <[email protected]>
  • Loading branch information
@jeandet
jeandet committed Nov 28, 2023
1 parent 0e490dc commit bc87314
Show file tree
Hide file tree
Showing 2 changed files with 173 additions and 56 deletions.
93 changes: 79 additions & 14 deletions scripts/direct_archive_yaml_gen.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2023-07-05T09:03:01.397104357Z",
"start_time": "2023-07-05T09:03:01.394156094Z"
Expand All @@ -18,6 +17,16 @@
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"ExecuteTime": {
"end_time": "2023-07-05T09:03:01.397594296Z",
"start_time": "2023-07-05T09:03:01.396060568Z"
},
"collapsed": false,
"jupyter": {
"outputs_hidden": false
}
},
"outputs": [],
"source": [
"inventory = {}\n",
Expand All @@ -35,35 +44,91 @@
"with open('../speasy/data/archive/themis_cdpp.yaml', 'w') as inv_f:\n",
" yaml.dump(inventory, inv_f)\n",
"\n"
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2023-07-05T09:03:01.397594296Z",
"start_time": "2023-07-05T09:03:01.396060568Z"
}
}
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"inventory = {}\n",
"\n",
"inventory['erg_lepe_l3_pa'] = {\n",
" 'url_pattern': 'https://cdaweb.gsfc.nasa.gov/pub/data/arase/lepe/l3/pa/{Y}/erg_lepe_l3_pa_{Y}{M:02d}{D:02d}_v\\\\d+_\\\\d+.cdf',\n",
" 'use_file_list': True,\n",
" 'master_cdf': \"https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/erg_lepe_l3_pa_00000000_v01.cdf\",\n",
" 'inventory_path': 'cda/Arase_ERG/LEPE',\n",
" 'split_rule': \"regular\"\n",
" }\n",
"\n",
"inventory['erg_pwe_hfa_l3_1min'] = {\n",
" 'url_pattern': 'https://cdaweb.gsfc.nasa.gov/pub/data/arase/pwe/hfa/l3_1min/{Y}/erg_pwe_hfa_l3_1min_{Y}{M:02d}{D:02d}_v\\\\d+_\\\\d+.cdf',\n",
" 'use_file_list': True,\n",
" 'master_cdf': \"https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/erg_pwe_hfa_l3_1min_00000000_v01.cdf\",\n",
" 'inventory_path': 'cda/Arase_ERG/PWE/HFA',\n",
" 'split_rule': \"regular\"\n",
" }\n",
"\n",
"for spacecraft in range(1,5):\n",
" for mode,MODE in (('fast','FAST'), ('brst','BURST')):\n",
" inventory[f'mms{spacecraft}_fpi_{mode}_l2_des_moms'] = {\n",
" 'url_pattern': f'https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms{spacecraft}/fpi/{mode}/l2/des-moms/{{Y}}/{{M:02d}}/mms{spacecraft}_fpi_{mode}_l2_des-moms_{{Y}}{{M:02d}}\\\\d+_v\\\\d+.\\\\d+.\\\\d+.cdf',\n",
" 'use_file_list': True,\n",
" 'master_cdf': f\"https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms{spacecraft}_fpi_{mode}_l2_des-moms_00000000_v01.cdf\",\n",
" 'inventory_path': f'cda/MMS/MMS{spacecraft}/FPI/{MODE}/MOMS',\n",
" 'split_rule': \"random\",\n",
" 'split_frequency': \"monthly\",\n",
" 'fname_regex': f'mms{spacecraft}_fpi_{mode}_l2_des-moms_(?P<start>\\\\d+)_v(?P<version>[\\\\d\\\\.]+)\\\\.cdf'\n",
" }\n",
"\n",
" inventory[f'mms{spacecraft}_fgm_srvy_l2'] = {\n",
" 'url_pattern': f'https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms{spacecraft}/fgm/srvy/l2/{{Y}}/{{M:02d}}/mms{spacecraft}_fgm_srvy_l2_{{Y}}{{M:02d}}{{D:02d}}_v\\\\d+.\\\\d+.\\\\d+.cdf',\n",
" 'use_file_list': True,\n",
" 'master_cdf': f\"https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms{spacecraft}_fgm_srvy_l2_00000000_v01.cdf\",\n",
" 'inventory_path': f'cda/MMS/MMS{spacecraft}/FGM/SRVY',\n",
" 'split_rule': \"regular\"\n",
" }\n",
"\n",
"\n",
"with open('../speasy/data/archive/cda.yaml', 'w') as inv_f:\n",
" yaml.dump(inventory, inv_f)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
"pygments_lexer": "ipython3",
"version": "3.12.0"
},
"widgets": {
"application/vnd.jupyter.widget-state+json": {
"state": {},
"version_major": 2,
"version_minor": 0
}
}
},
"nbformat": 4,
"nbformat_minor": 0
"nbformat_minor": 4
}
136 changes: 94 additions & 42 deletions speasy/data/archive/cda.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,48 +1,100 @@
---
erg_lepe_l3_pa:
url_pattern: "https://cdaweb.gsfc.nasa.gov/pub/data/arase/lepe/l3/pa/{Y}/erg_lepe_l3_pa_{Y}{M:02d}{D:02d}_v03_01.cdf"
master_cdf: "https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/erg_lepe_l3_pa_00000000_v01.cdf"
inventory_path: 'cda/Arase_ERG/LEPE'
split_rule: "regular"
inventory_path: cda/Arase_ERG/LEPE
master_cdf: https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/erg_lepe_l3_pa_00000000_v01.cdf
split_rule: regular
url_pattern: https://cdaweb.gsfc.nasa.gov/pub/data/arase/lepe/l3/pa/{Y}/erg_lepe_l3_pa_{Y}{M:02d}{D:02d}_v\d+_\d+.cdf
use_file_list: true
erg_pwe_hfa_l3_1min:
url_pattern: 'https://cdaweb.gsfc.nasa.gov/pub/data/arase/pwe/hfa/l3_1min/{Y}/erg_pwe_hfa_l3_1min_{Y}{M:02d}{D:02d}_v\d+_\d+.cdf'
use_file_list: true
master_cdf: "https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/erg_pwe_hfa_l3_1min_00000000_v01.cdf"
inventory_path: 'cda/Arase_ERG/PWE/HFA'
split_rule: "regular"

inventory_path: cda/Arase_ERG/PWE/HFA
master_cdf: https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/erg_pwe_hfa_l3_1min_00000000_v01.cdf
split_rule: regular
url_pattern: https://cdaweb.gsfc.nasa.gov/pub/data/arase/pwe/hfa/l3_1min/{Y}/erg_pwe_hfa_l3_1min_{Y}{M:02d}{D:02d}_v\d+_\d+.cdf
use_file_list: true
mms1_fgm_srvy_l2:
inventory_path: cda/MMS/MMS1/FGM/SRVY
master_cdf: https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms1_fgm_srvy_l2_00000000_v01.cdf
split_rule: regular
url_pattern: https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms1/fgm/srvy/l2/{Y}/{M:02d}/mms1_fgm_srvy_l2_{Y}{M:02d}{D:02d}_v\d+.\d+.\d+.cdf
use_file_list: true
mms1_fpi_brst_l2_des_moms:
url_pattern: 'https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms1/fpi/brst/l2/des-moms/{Y}/{M:02d}/mms1_fpi_brst_l2_des-moms_{Y}{M:02d}\d+_v\d+.\d+.\d+.cdf'
use_file_list: true
master_cdf: "https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms1_fpi_brst_l2_des-moms_00000000_v01.cdf"
inventory_path: 'cda/MMS/MMS1/FPI/BURST/MOMS'
split_rule: "random"
split_frequency: "monthly"
fname_regex: 'mms1_fpi_brst_l2_des-moms_(?P<start>\d+)_v(?P<version>[\d\.]+)\.cdf'

fname_regex: mms1_fpi_brst_l2_des-moms_(?P<start>\d+)_v(?P<version>[\d\.]+)\.cdf
inventory_path: cda/MMS/MMS1/FPI/BURST/MOMS
master_cdf: https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms1_fpi_brst_l2_des-moms_00000000_v01.cdf
split_frequency: monthly
split_rule: random
url_pattern: https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms1/fpi/brst/l2/des-moms/{Y}/{M:02d}/mms1_fpi_brst_l2_des-moms_{Y}{M:02d}\d+_v\d+.\d+.\d+.cdf
use_file_list: true
mms1_fpi_fast_l2_des_moms:
fname_regex: mms1_fpi_fast_l2_des-moms_(?P<start>\d+)_v(?P<version>[\d\.]+)\.cdf
inventory_path: cda/MMS/MMS1/FPI/FAST/MOMS
master_cdf: https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms1_fpi_fast_l2_des-moms_00000000_v01.cdf
split_frequency: monthly
split_rule: random
url_pattern: https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms1/fpi/fast/l2/des-moms/{Y}/{M:02d}/mms1_fpi_fast_l2_des-moms_{Y}{M:02d}\d+_v\d+.\d+.\d+.cdf
use_file_list: true
mms2_fgm_srvy_l2:
inventory_path: cda/MMS/MMS2/FGM/SRVY
master_cdf: https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms2_fgm_srvy_l2_00000000_v01.cdf
split_rule: regular
url_pattern: https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms2/fgm/srvy/l2/{Y}/{M:02d}/mms2_fgm_srvy_l2_{Y}{M:02d}{D:02d}_v\d+.\d+.\d+.cdf
use_file_list: true
mms2_fpi_brst_l2_des_moms:
url_pattern: 'https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms2/fpi/brst/l2/des-moms/{Y}/{M:02d}/mms2_fpi_brst_l2_des-moms_{Y}{M:02d}\d+_v\d+.\d+.\d+.cdf'
use_file_list: true
master_cdf: "https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms2_fpi_brst_l2_des-moms_00000000_v01.cdf"
inventory_path: 'cda/MMS/MMS2/FPI/BURST/MOMS'
split_rule: "random"
split_frequency: "monthly"
fname_regex: 'mms2_fpi_brst_l2_des-moms_(?P<start>\d+)_v(?P<version>[\d\.]+)\.cdf'

fname_regex: mms2_fpi_brst_l2_des-moms_(?P<start>\d+)_v(?P<version>[\d\.]+)\.cdf
inventory_path: cda/MMS/MMS2/FPI/BURST/MOMS
master_cdf: https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms2_fpi_brst_l2_des-moms_00000000_v01.cdf
split_frequency: monthly
split_rule: random
url_pattern: https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms2/fpi/brst/l2/des-moms/{Y}/{M:02d}/mms2_fpi_brst_l2_des-moms_{Y}{M:02d}\d+_v\d+.\d+.\d+.cdf
use_file_list: true
mms2_fpi_fast_l2_des_moms:
fname_regex: mms2_fpi_fast_l2_des-moms_(?P<start>\d+)_v(?P<version>[\d\.]+)\.cdf
inventory_path: cda/MMS/MMS2/FPI/FAST/MOMS
master_cdf: https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms2_fpi_fast_l2_des-moms_00000000_v01.cdf
split_frequency: monthly
split_rule: random
url_pattern: https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms2/fpi/fast/l2/des-moms/{Y}/{M:02d}/mms2_fpi_fast_l2_des-moms_{Y}{M:02d}\d+_v\d+.\d+.\d+.cdf
use_file_list: true
mms3_fgm_srvy_l2:
inventory_path: cda/MMS/MMS3/FGM/SRVY
master_cdf: https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms3_fgm_srvy_l2_00000000_v01.cdf
split_rule: regular
url_pattern: https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms3/fgm/srvy/l2/{Y}/{M:02d}/mms3_fgm_srvy_l2_{Y}{M:02d}{D:02d}_v\d+.\d+.\d+.cdf
use_file_list: true
mms3_fpi_brst_l2_des_moms:
url_pattern: 'https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms3/fpi/brst/l2/des-moms/{Y}/{M:02d}/mms3_fpi_brst_l2_des-moms_{Y}{M:02d}\d+_v\d+.\d+.\d+.cdf'
use_file_list: true
master_cdf: "https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms3_fpi_brst_l2_des-moms_00000000_v01.cdf"
inventory_path: 'cda/MMS/MMS3/FPI/BURST/MOMS'
split_rule: "random"
split_frequency: "monthly"
fname_regex: 'mms3_fpi_brst_l2_des-moms_(?P<start>\d+)_v(?P<version>[\d\.]+)\.cdf'

fname_regex: mms3_fpi_brst_l2_des-moms_(?P<start>\d+)_v(?P<version>[\d\.]+)\.cdf
inventory_path: cda/MMS/MMS3/FPI/BURST/MOMS
master_cdf: https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms3_fpi_brst_l2_des-moms_00000000_v01.cdf
split_frequency: monthly
split_rule: random
url_pattern: https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms3/fpi/brst/l2/des-moms/{Y}/{M:02d}/mms3_fpi_brst_l2_des-moms_{Y}{M:02d}\d+_v\d+.\d+.\d+.cdf
use_file_list: true
mms3_fpi_fast_l2_des_moms:
fname_regex: mms3_fpi_fast_l2_des-moms_(?P<start>\d+)_v(?P<version>[\d\.]+)\.cdf
inventory_path: cda/MMS/MMS3/FPI/FAST/MOMS
master_cdf: https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms3_fpi_fast_l2_des-moms_00000000_v01.cdf
split_frequency: monthly
split_rule: random
url_pattern: https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms3/fpi/fast/l2/des-moms/{Y}/{M:02d}/mms3_fpi_fast_l2_des-moms_{Y}{M:02d}\d+_v\d+.\d+.\d+.cdf
use_file_list: true
mms4_fgm_srvy_l2:
inventory_path: cda/MMS/MMS4/FGM/SRVY
master_cdf: https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms4_fgm_srvy_l2_00000000_v01.cdf
split_rule: regular
url_pattern: https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms4/fgm/srvy/l2/{Y}/{M:02d}/mms4_fgm_srvy_l2_{Y}{M:02d}{D:02d}_v\d+.\d+.\d+.cdf
use_file_list: true
mms4_fpi_brst_l2_des_moms:
url_pattern: 'https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms4/fpi/brst/l2/des-moms/{Y}/{M:02d}/mms4_fpi_brst_l2_des-moms_{Y}{M:02d}\d+_v\d+.\d+.\d+.cdf'
use_file_list: true
master_cdf: "https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms4_fpi_brst_l2_des-moms_00000000_v01.cdf"
inventory_path: 'cda/MMS/MMS4/FPI/BURST/MOMS'
split_rule: "random"
split_frequency: "monthly"
fname_regex: 'mms4_fpi_brst_l2_des-moms_(?P<start>\d+)_v(?P<version>[\d\.]+)\.cdf'
fname_regex: mms4_fpi_brst_l2_des-moms_(?P<start>\d+)_v(?P<version>[\d\.]+)\.cdf
inventory_path: cda/MMS/MMS4/FPI/BURST/MOMS
master_cdf: https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms4_fpi_brst_l2_des-moms_00000000_v01.cdf
split_frequency: monthly
split_rule: random
url_pattern: https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms4/fpi/brst/l2/des-moms/{Y}/{M:02d}/mms4_fpi_brst_l2_des-moms_{Y}{M:02d}\d+_v\d+.\d+.\d+.cdf
use_file_list: true
mms4_fpi_fast_l2_des_moms:
fname_regex: mms4_fpi_fast_l2_des-moms_(?P<start>\d+)_v(?P<version>[\d\.]+)\.cdf
inventory_path: cda/MMS/MMS4/FPI/FAST/MOMS
master_cdf: https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/mms4_fpi_fast_l2_des-moms_00000000_v01.cdf
split_frequency: monthly
split_rule: random
url_pattern: https://cdaweb.gsfc.nasa.gov/pub/data/mms/mms4/fpi/fast/l2/des-moms/{Y}/{M:02d}/mms4_fpi_fast_l2_des-moms_{Y}{M:02d}\d+_v\d+.\d+.\d+.cdf
use_file_list: true

0 comments on commit bc87314

Please sign in to comment.