Updates to enable the MeDDEA simulator (#1)

* update to docs with logo and putting back old docs

* fix to log file name

* added readthedocs docs badge and added upload to codecov.io

* added data directory for easy access to init

* Create spectrum.py

* fixed python version for readthedocs

* fix to readthedocs config

* added code to simulate flare and ba133 photons

* added the ability to simulate x-ray sources

* added file reading tools

* added ccsdspy as dependency

* added ability to put photon in pixels for simulations

.github/workflows/test_in_container.yml

@@ -23,5 +23,6 @@ jobs:
       run: pytest --pyargs padre_meddea --cov padre_meddea
       env:
         PLATFORM: 'docker'
-    - name: Upload coverage reports to Codecov with GitHub Action
-      uses: codecov/codecov-action@v3
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v3
+        env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -176,4 +176,12 @@ Icon
 .AppleDesktop
 Network Trash Folder
 Temporary Items
-.apdisk
+.apdisk
+
+# jupyter notebooks should not be added to Python Packages
+*.ipynb
+
+*.bin
+_version.py
+
+*.csv

.readthedocs.yml

@@ -1,23 +1,27 @@
-# .readthedocs.yml
-# Read the Docs configuration file
+# Read the Docs configuration file for Sphinx projects
 # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
 
 # Required
 version: 2
 
-# Build documentation in the docs/ directory with Sphinx
-sphinx:
-  configuration: docs/conf.py
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
 
 # Optionally build your docs in additional formats such as PDF and ePub
-formats: []
+formats:
+  - pdf
+  - epub
 
-# Optionally set the version of Python and requirements required to build your docs
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
 python:
-  version: 3.8
   install:
     - method: pip
       path: .
       extra_requirements:
         - docs
-        - all
+        - all

README.rst

@@ -8,9 +8,10 @@ Overview
     :stub-columns: 1
 
     * - docs
-      - |docs|
+      - |docs| |readthedocs|
     * - build status
       - |testing| |codestyle|
+
 .. |docs| image:: https://github.com/PADRESat/padre_meddea/actions/workflows/docs.yml/badge.svg
     :target: https://github.com/PADRESat/padre_meddea/actions/workflows/docs.yml
     :alt: Documentation Build Status
@@ -23,6 +24,10 @@ Overview
     :target: https://github.com/PADRESat/padre_meddea/actions/workflows/codestyle.yml
     :alt: Codestyle and linting using flake8
 
+.. |readthedocs| image:: https://readthedocs.org/projects/padre-meddea/badge/?version=latest
+    :target: https://padre-meddea.readthedocs.io/en/latest/?badge=latest
+    :alt: Documentation Status
+
 .. end-badges
 
 A Python package to process and analyze data from the Measuring Directivity to Determine Electron Anisotropy (MeDDEA) instrument on the 

docs/cmad/related_docs.rst

@@ -20,10 +20,29 @@ level guiding requirements or that provide more detail or context. See example b
     * - `Heliophysics Division Science Data Management Policy <https://science.nasa.gov/science-pink/s3fs-public/atoms/files/HPD%20Data%20Policy_Final_20220209.pdf>`_
       - HPD-SDMP 
       - 14 Feb 2022
+    * - `Scientific Information policy for the Science Mission Directorate <https://science.nasa.gov/science-red/s3fs-public/atoms/files/Scientific%20Information%20policy%20SPD-41.pdf>`_
+      - SPD-41
+      - 4 Aug, 2021
     * - PADRE MEDDEA Project Level Requirement Appendix (PLRA)   
       - PADRE MEDDEA-SYS-REQ-0027 
       - 13 Oct 2021 
     * - PADRE MEDDEA Project Data Management Plan (PDMP)
       - MGMT-PLAN-0015
       - 7 Oct 2021
 
+Applicable Software
+-------------------
+
+.. list-table::
+    :widths: 50 25 50
+    :header-rows: 1
+
+    * - Title
+      - Purpose
+      - Repository
+    * - stixpy
+      - Analysis software for Solar Orbiter Spectrometer Telescope for Imaging X-rays
+      - https://github.com/samaloney/stixpy 
+    * - stixCore
+      - STIX Core Utilities for the STIX Science Operations Center      
+      - https://github.com/i4Ds/STIXCore

docs/conf.py

@@ -12,8 +12,8 @@
 # -- Project information -----------------------------------------------------
 
 project = "padre_meddea"
-copyright = ""
-author = "The PADRE MEDDEA Team"
+copyright = "US Government (not copyrighted) 2023"
+author = "The PADRE MeDDEA Team"
 
 # The full version, including alpha/beta/rc tags
 from padre_meddea import __version__
@@ -39,8 +39,12 @@
     "sphinx.ext.mathjax",
     "sphinx_automodapi.automodapi",
     "sphinx_automodapi.smart_resolver",
+    "matplotlib.sphinxext.plot_directive"
 ]
 
+# plot_directive default to always show source when including a plot
+plot_include_source = True
+
 # Set automodapi to generate files inside the generated directory
 # automodapi_toctreedirnm = "_build/html/api"
 numpydoc_show_class_members = False
@@ -104,11 +108,9 @@
 html_theme = "bizstyle"
 html_static_path = ["_static"]
 
-#html_logo = "logo/MEDDEA_logo.png"
+html_logo = "logo/padre_logo.png"
 #html_favicon = "logo/favicon.ico"
-#html_css_files = [
-#    "css/custom.css",
-#]
+html_css_files = ["css/custom.css"]
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".

docs/user-guide/customization.rst

@@ -43,4 +43,4 @@ A sample configrc file
 
     `(download) <../_static/configrc>`__
 
-.. literalinclude:: ../../padre_meddea_eea/data/configrc
+.. literalinclude:: ../../padre_meddea/data/configrc

docs/user-guide/data.rst

@@ -9,29 +9,88 @@ Overview
 
 Data Description
 ----------------
+MeDDEA has two primary data products which originate from the same measurements.
+
+#. an x-ray spectrum, provided regularly and 
+#. a x-ray photon list provided on-demand.
+
+Generally, the photon list data product will only exist during large flares and calibration periods.
+It is generated automatically on board the spacecraft but only downloaded when requested.
+This photon list will be used to generate an x-ray spectrum that supersedes the histogram data product.
+
+The data levels for the spectrum product are described below.
+These products are processed on the ground using this Python package unless otherwise specified.
 
 +----------+---------------------------------------+---------------------------------------+
 | Level    | Product                               | Description                           |      
 +==========+=======================================+=======================================+
-| 1        | Vector magnetic field from 3 sensors. | CCSDS, each packet is 4-sec long      |
-|          |                                       | at 10 Hz rate, 3-axis magnetic        |
-|          |                                       | field components for all three sensors|
+| 1        | Count Spectrum in energy space        | FITS file, produced at least every 1 s|
+|          | integrated across all pixels and      | , generated on the spacecraft         |
+|          | detectors                             |                                       |
 +----------+---------------------------------------+---------------------------------------+
 
+The data levels for the photon list product are described below.
 
++----------+---------------------------------------+---------------------------------------+
+| Level    | Product                               | Description                           |      
++==========+=======================================+=======================================+
+| 0        | List of hits. Each photon has         | FITS file, file will consist of a     |
+|          | relative time, photon energy in ADC   | fixed number of hits                  |
+|          | counts and pixel and detector number  | , generated on the spacecraft         |
++----------+---------------------------------------+---------------------------------------+
+| 1        | List of photons. Each photon has      | FITS file, produced with a fixed      |
+|          | time of arrival in UTC and calibrated | number photons and variable           |                       
+|          | energy for each pixel and detector    | integration times.                    |
++----------+---------------------------------------+---------------------------------------+
 
+Both of the above products can be used to generate a calibrated spectrum product.
 
++----------+---------------------------------------+---------------------------------------+
+| Level    | Product                               | Description                           |      
++==========+=======================================+=======================================+
+| 2        | Flux Spectrum in energy space,        | FITS, data flag to state if it was    |
+|          | integrated over all detectors and     | generated from the photon list or not |
+|          | pixels                                |                                       |
++----------+---------------------------------------+---------------------------------------+
 
+The above data product will be used to generate the following derived data products.
 
-Getting Data
-============
++----------+---------------------------------------+---------------------------------------+
+| Level    | Product                               | Description                           |      
++==========+=======================================+=======================================+
+| 3        | Flare X-ray Directivity as a function | FITS file, requires Solar Orbiter STIX|
+|          | of energy and time for the angular    | data, ratio of STIX to PADRE flux     |
+|          | separation between STIX and PADRE     |                                       |
++----------+---------------------------------------+---------------------------------------+
+| 4        | Flare-accelerated Electron Anisotropy | FITS file, requires modeling analysis |
+|          | as a function of energy and time.     |                                       |
++----------+---------------------------------------+---------------------------------------+
 
+File Naming Conventions
+-----------------------
 
+The file naming conventions for the products listed above are
 
-Reading Data
+#. padre_meddea_l0_photonlist_%Y%m%d_v{version}
+#. padre_meddea_l1_spec_%Y%m%d_v{version}
+#. padre_meddea_l2_spec_%Y%m%d_v{version}
+#. padre_meddea_l3_xraydirect_%Y%m%d_{angle}_v{version}
+#. padre_meddea_l4_eanisotropy_%Y%m%d_v{version}
+
+Files will be generated daily and include a full day of data.
+The `{version}` begins at one and is incremented everytime the data file is updated.
+The `{angle}` is the average angular separation between STIX and PADRE at the time of the observation.
+If too large, a file may be split up into multiple files in which case the date string will add the hour and minute and the integration time.
+Specified using Python datetime strftime definitions and version is a 3 digit zero-padded number which begins at 000 and increments every time the file is reprocessed.
+See `Python strftime cheatsheet <https://strftime.org/>`_ for a quick reference.
+
+Getting Data
 ============
 
+To be written.
 
+Reading Data
+============
 
 Calibrating Data
 ================

docs/user-guide/index.rst

@@ -12,5 +12,7 @@ For more details checkout the :ref:`reference`.
 
    Brief Tour <tour>
    data
+   level0
+   spectrum
    customization
    logger

docs/user-guide/level0.rst

@@ -0,0 +1,41 @@
+.. _level0:
+
+************
+Level 0 Data
+************
+
+Overview
+========
+This section document the format of the level 0 binary files.
+Level 0 binary files are raw telemetry or command response packets generated by the instrument.
+
+
+Photon Packet
+-------------
+The packet which provides individual hit data or photons for each detector.
+
++-------------+-----------+--------+------------------------------------------------------------+
+| Name        | Bit Size  | Type   | Description                                                |
++=============+===========+========+============================================================+
+| HEADER WORD | 16        | UINT   | value is always 65131                                      |
++-------------+-----------+--------+------------------------------------------------------------+
+| FLAGS       | 16        | UNIT   | various flags, To be defined                               |
++-------------+-----------+--------+------------------------------------------------------------+
+| Packet Size | 16        | UINT   | The size of the packet which can be used to determine the  |
+|             |           |        | number of hits included in the packet                      |
++-------------+-----------+--------+------------------------------------------------------------+
+| Time Stamp  | 48        | UINT   |                                                            |
++-------------+-----------+--------+------------------------------------------------------------+
+| Checksum    | 16        | UINT   | For data integrity                                         |
++-------------+-----------+--------+------------------------------------------------------------+
+| Pixel data  |           |        | This field is repeated based on the number of hits detected|
++-------------+-----------+--------+------------------------------------------------------------+
+
+
++-------------+-----------+--------+------------------------------------------------------------+
+| Pixel Data  |           |        |                                                            |
++-------------+-----------+--------+------------------------------------------------------------+
+| Detector id | 16        | UINT   | The detector id for the location of the hit                |
++-------------+-----------+--------+------------------------------------------------------------+
+| Hit energy  | 16        | UINT   | The ADC value for the energy of the hit                    |
++-------------+-----------+--------+------------------------------------------------------------+

docs/user-guide/spectrum.rst

@@ -0,0 +1,83 @@
+.. _spectrum,:
+
+**************
+Spectrum Tools
+**************
+
+Overview
+========
+This section tools to simulate and analyze spectra.
+
+
+Simulated Data
+--------------
+This package provides tools to simulate the data that is expected during science
+operations.
+
+.. plot::
+
+    A plot of the sample X class flare time series (SOL2002-07-23):
+
+    >>> import numpy as np
+    >>> import matplotlib.pyplot as plt
+    >>> import astropy.units as u
+    >>> import padre_meddea.calibration.spectrum
+    >>> from padre_meddea.calibration.spectrum import get_flare_rate, flare_timeseries
+    >>> time = np.arange(10000) * u.s
+    >>> cts = get_flare_rate()(time)
+    >>> data_limiter = flare_timeseries['sec_from_start'] < time.max()
+    >>> y = flare_timeseries['xrsb'][data_limiter] / flare_timeseries['xrsb'].max() * 22421.0
+    >>> total_counts = cts.sum() * 4
+    >>> plt.plot(flare_timeseries['sec_from_start'][data_limiter], y, label='data [GOES xrbs]')
+    >>> plt.plot(time, cts, label='Interpolated', alpha=0.5, linewidth=3.0)
+    >>> plt.legend()
+    >>> plt.ylabel('cts/s/detector')
+    >>> plt.xlabel('seconds from start')
+    >>> plt.title(f'SOL2002-07-23 {total_counts/1e6:.2f} Mcounts')
+
+.. plot::
+
+    A plot of the spectrum of our sample X class flare (SOL2002-07-23):
+
+    >>> import numpy as np
+    >>> import matplotlib.pyplot as plt
+    >>> import astropy.units as u
+    >>> import padre_meddea.calibration.spectrum
+    >>> from padre_meddea.calibration.spectrum import flare_spectrum, flare_spectrum_data
+    >>> energy = np.arange(5, 100, 1) * u.keV
+    >>> flux = flare_spectrum(1)(energy)
+    >>> data_limiter = flare_spectrum_data['Bin mean (keV)'] < energy.max()
+    >>> y = flare_spectrum_data['Flux (photon/s/cm ** 2/keV)'][data_limiter]
+    >>> plt.plot(flare_spectrum_data['Bin mean (keV)'][data_limiter], y, label='data [RHESSI]')
+    >>> plt.plot(energy, flux, label='Interpolated', alpha=0.5, linewidth=3.0)
+    >>> plt.legend()
+    >>> plt.yscale('log')
+    >>> plt.xscale('log')
+    >>> plt.ylabel('photon/s/cm ** 2/keV')
+    >>> plt.xlabel('energy [keV]')
+    >>> plt.title('SOL2002-07-23')
+
+.. plot::
+
+    A plot of our Ba-133 calibration source:
+
+    >>> import numpy as np
+    >>> import matplotlib.pyplot as plt
+    >>> import astropy.units as u
+    >>> import padre_meddea.calibration.spectrum
+    >>> from padre_meddea.calibration.spectrum import barium_spectrum, ba133_lines, cd_kalpha1, te_kalpha1
+    >>> energy = np.arange(5, 150, 0.3) * u.keV
+    >>> flux = barium_spectrum(fwhm = 1 * u.keV)(energy)
+    >>> plt.plot(energy, flux, label='Simulated', alpha=0.5, linewidth=3.0)
+    >>> for i, this_line in enumerate(ba133_lines):
+    >>>     cd_escape_line = this_line['energy (eV)'] - cd_kalpha1
+    >>>     te_escape_line = this_line['energy (eV)'] - te_kalpha1
+    >>>     if cd_escape_line > 0 * u.eV:
+    >>>         plt.axvline(x=cd_escape_line.to('keV').value, label=f'{cd_escape_line.to("keV"):.2f} Cd escape', color='red')
+    >>>     if te_escape_line > 0 * u.eV:
+    >>>         plt.axvline(x=te_escape_line.to('keV').value, label=f'{te_escape_line.to("keV"):.2f} Te escape', color='red')
+    >>>     plt.axvline(x=this_line['energy (eV)'].to('keV').value, label=f'{this_line["energy (eV)"].to("keV"):.2f} {this_line["name"]}')
+    >>> plt.legend(loc='center left', bbox_to_anchor=(0.60, 0.5))
+    >>> plt.ylabel('')
+    >>> plt.xlabel('energy [keV]')
+    >>> plt.title('Ba-133 Calibration source')