diff --git a/Makefile b/Makefile index d4d88b46a..84a9c3331 100644 --- a/Makefile +++ b/Makefile @@ -27,13 +27,7 @@ $(HEXBIN_OUT): $(HEXBIN_IN) yamllint: $(YAML_FILES_OUT) -data_table: _data/data_table.py - python _data/data_table.py - -doi_data: _data/to_doi_csv.py - python _data/to_doi_csv.py - -simulations: yamllint data_table +simulations: yamllint hexbin: $(HEXBIN_OUT) diff --git a/_data/charts/benchmark_upload.json b/_data/charts/benchmark_upload.json index 57b2cf8d0..e8a75fbbb 100644 --- a/_data/charts/benchmark_upload.json +++ b/_data/charts/benchmark_upload.json @@ -51,97 +51,6 @@ "values": [] } ], - "data": [ - { - "name": "code_uploads", - "values": [ - { - "count": 17, - "name": "1b" - }, - { - "count": 13, - "name": "1c" - }, - { - "count": 12, - "name": "1a" - }, - { - "count": 10, - "name": "3a" - }, - { - "count": 8, - "name": "1d" - }, - { - "count": 7, - "name": "2a" - }, - { - "count": 6, - "name": "2b" - }, - { - "count": 6, - "name": "2c" - }, - { - "count": 6, - "name": "2d" - }, - { - "count": 6, - "name": "7a" - }, - { - "count": 5, - "name": "6a" - }, - { - "count": 3, - "name": "6b" - }, - { - "count": 1, - "name": "4a" - }, - { - "count": 1, - "name": "4b" - }, - { - "count": 1, - "name": "4c" - }, - { - "count": 1, - "name": "4d" - }, - { - "count": 1, - "name": "4e" - }, - { - "count": 1, - "name": "4f" - }, - { - "count": 1, - "name": "4g" - }, - { - "count": 1, - "name": "4h" - }, - { - "count": 1, - "name": "7b" - } - ] - } - ], "height": 115, "marks": [ { @@ -262,4 +171,4 @@ } ], "width": 230 -} \ No newline at end of file +} diff --git a/_data/charts/code_upload.json b/_data/charts/code_upload.json index e34d8873f..04aeeabb4 100644 --- a/_data/charts/code_upload.json +++ b/_data/charts/code_upload.json @@ -51,49 +51,6 @@ "values": [] } ], - "data": [ - { - "name": "code_uploads", - "values": [ - { - "count": 58, - "name": "moose" - }, - { - "count": 10, - "name": "mmsp" - }, - { - "count": 9, - "name": "prisms" - }, - { - "count": 8, - "name": "fenics" - }, - { - "count": 8, - "name": "sfepy" - }, - { - "count": 7, - "name": "fipy" - }, - { - "count": 4, - "name": "custom" - }, - { - "count": 3, - "name": "matlab" - }, - { - "count": 1, - "name": "hiperc" - } - ] - } - ], "height": 115, "marks": [ { @@ -214,4 +171,4 @@ } ], "width": 230 -} \ No newline at end of file +} diff --git a/_data/data_table.py b/_data/data_table.py deleted file mode 100644 index b520a0326..000000000 --- a/_data/data_table.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Data pipeline to build data for simulation data table. -""" -# pylint: disable=no-value-for-parameter - -import os - -import yaml - -# pylint: disable=redefined-builtin, no-name-in-module -from toolz.curried import pipe, curry, map, groupby, get, valmap, count - -from simulations import get_path, render_yaml, get_yaml_data, j2_to_json - - -def table_yaml(): - """Path to table YAML. - """ - return os.path.join(get_path(), "table.yaml.j2") - - -@curry -def write_yaml(data, filepath): - """Write yaml - """ - with open(filepath, "w") as stream: - yaml.dump(data, stream, indent=2) - return (filepath, data) - - -def make_table_yaml(): - """Make simulation datatable from meta.yaml's. - """ - return pipe( - get_yaml_data(), - lambda data: render_yaml(table_yaml(), data=data), - yaml.load, - write_yaml(filepath=os.path.join(get_path(), "../_data/data_table.yaml")), - ) - - -def groupby_count(func): - """Group the simulation data based on a function. - - Args: - func: function to group by - - Returns: - grouped data - """ - return pipe(get_yaml_data(), map(get(1)), groupby(func), valmap(count)) - - -def code_upload_yaml_path(): - """Path to the upload chart j2. - """ - return os.path.join(get_path(), "charts", "code_upload.yaml.j2") - - -def make_upload_chart(gfunc, yaml_path, json_path, title): - """Create an upload chart - - Args: - gfunc: the group by function - yaml_path: the template path - json_path: the path to write to - title: the title of the chart - - Returns: - the chart JSON - """ - return pipe( - gfunc, - groupby_count, - lambda data: list(data.items()), - lambda data: sorted(data, key=lambda item: (-item[1], item[0])), - lambda data: j2_to_json(yaml_path, json_path, data=data, title=title), - ) - - -if __name__ == "__main__": - make_table_yaml() - make_upload_chart( - lambda item: item["metadata"]["implementation"]["name"], - code_upload_yaml_path(), - os.path.join(get_path(), "../_data/charts/code_upload.json"), - "Uploads per Code", - ) - - make_upload_chart( - lambda item: item["benchmark"]["id"], - code_upload_yaml_path(), - os.path.join(get_path(), "../_data/charts/benchmark_upload.json"), - "Uploads per Benchmark", - ) diff --git a/_data/data_table.yaml b/_data/data_table.yaml deleted file mode 100644 index 0b0bef5f4..000000000 --- a/_data/data_table.yaml +++ /dev/null @@ -1,649 +0,0 @@ -data: -- author: {email: cmskdu@gmail.com, name: Dong-Uk Kim} - code: {name: moose, url: 'https://github.com/eastglow-zz/CHiMaD3_dendritic_growth'} - cores: 8 - id_: 3a.1 - name: CHiMaD3_MOOSE - timestamp: 2018/06/27 00:00:00 -- author: {email: pusztai.tamas@wigner.mta.hu, name: "Tam\xE1s Pusztai"} - code: {name: matlab, url: 'https://mydrive.kfki.hu/f/8565830aa4f24610922a/?dl=1'} - cores: 3584 - id_: 3a.1 - name: Finite_Diff_A - timestamp: 2018/10/06 00:00:00 -- author: {email: pusztai.tamas@wigner.mta.hu, name: "Tam\xE1s Pusztai"} - code: {name: matlab, url: 'https://mydrive.kfki.hu/f/de47ed05b4e74e46ad80/?dl=1'} - cores: 3584 - id_: 3a.1 - name: Finite_Diff_B - timestamp: 2018/10/06 00:00:00 -- author: {email: pusztai.tamas@wigner.mta.hu, name: "Tam\xE1s Pusztai"} - code: {name: matlab, url: 'https://mydrive.kfki.hu/f/8565830aa4f24610922a/?dl=1'} - cores: 3584 - id_: 3a.1 - name: Finite_Diff_C - timestamp: 2018/10/06 00:00:00 -- author: {email: andrembc10@gmail.com, name: Andre M. B. Costa} - code: {name: custom, url: 'https://github.com/andrembcosta/1st-Order-SI-3a'} - cores: 1 - id_: 3a.1 - name: FirstOrder_Semi_Implicit - timestamp: 2018/05/15 00:00:00 -- author: {email: narut.se@gmail.com, name: Narut Sereewattanawoot} - code: {name: custom, url: 'https://github.com/narutse/phasefield'} - cores: 1 - id_: 7a.0 - name: Forward_Backward_Euler - timestamp: 2018/02/23 00:00:00 -- author: {email: stvdwtt@umich.edu, name: Stephen DeWitt} - code: {name: prisms, url: 'https://github.com/prisms-center/phaseField/tree/master/applications/CHiMaD_benchmark7'} - cores: 6 - id_: 7a.0 - name: PRISMSPF_Linear - timestamp: 2018/02/07 00:00:00 -- author: {email: stvdwtt@umich.edu, name: Stephen DeWitt} - code: {name: prisms, url: 'https://github.com/prisms-center/phaseField/tree/master/applications/CHiMaD_benchmark7'} - cores: 6 - id_: 7a.0 - name: PRISMSPF_Quad - timestamp: 2018/02/07 00:00:00 -- author: {email: stvdwtt@umich.edu, name: Stephen DeWitt} - code: {name: prisms, url: 'https://github.com/prisms-center/phaseField/tree/master/applications/CHiMaD_benchmark7'} - cores: 6 - id_: 7b.0 - name: PRISMSPF_Quad_7b - timestamp: 2018/02/01 00:00:00 -- author: {email: stvdwtt@umich.edu, name: Stephen DeWitt} - code: {name: prisms, url: 'https://github.com/prisms-center/phaseField'} - cores: 6 - id_: 3a.1 - name: PRISMS_PF_3a_cubic - timestamp: 2018/08/27 00:00:00 -- author: {email: stvdwtt@umich.edu, name: Stephen DeWitt} - code: {name: prisms, url: 'https://github.com/stvdwtt/phaseField/tree/bm3_speed_test'} - cores: 6 - id_: 3a.1 - name: PRISMS_PF_3a_faster - timestamp: 2018/10/12 00:00:00 -- author: {email: michael.tonks@ufl.edu, name: Michael Tonks} - code: {name: moose, url: 'https://github.com/idaholab/moose'} - cores: 4 - id_: 7a.0 - name: Tonks_MMS_results - timestamp: 2018/02/16 00:00:00 -- author: {email: andrea.jokisaari@northwestern.edu, name: A. M. Jokisaari} - code: {name: moose, url: 'https://www.dropbox.com/s/qusutz8g24hx54q/free_energy.csv'} - cores: 6 - id_: 6a.0 - name: coral_6a - timestamp: 2017/07/31 00:00:00 -- author: {email: ivan.yashchuk@vtt.fi, name: Ivan Yashchuk} - code: {name: fenics, url: 'https://bitbucket.org/fenics-project/dolfin/'} - cores: 24 - id_: 1a.1 - name: fenics_1a_ivan - timestamp: 2018/11/09 00:00:00 -- author: {email: ivan.yashchuk@vtt.fi, name: Ivan Yashchuk} - code: {name: fenics, url: 'https://bitbucket.org/fenics-project/dolfin/'} - cores: 24 - id_: 1b.1 - name: fenics_1b_ivan - timestamp: 2018/11/09 00:00:00 -- author: {email: ivan.yashchuk@vtt.fi, name: Ivan Yashchuk} - code: {name: fenics, url: 'https://bitbucket.org/fenics-project/dolfin/'} - cores: 24 - id_: 1c.1 - name: fenics_1c_ivan - timestamp: 2018/11/09 00:00:00 -- author: {email: ivan.yashchuk@vtt.fi, name: Ivan Yashchuk} - code: {name: fenics, url: 'https://bitbucket.org/fenics-project/dolfin/'} - cores: 24 - id_: 1d.1 - name: fenics_1d_ivan - timestamp: 2018/11/09 00:00:00 -- author: {email: ivan.yashchuk@vtt.fi, name: Ivan Yashchuk} - code: {name: fenics, url: 'https://bitbucket.org/fenics-project/dolfin/'} - cores: 24 - id_: 2a.1 - name: fenics_2a_ivan - timestamp: 2018/11/09 00:00:00 -- author: {email: ivan.yashchuk@aalto.fi, name: Ivan Yashchuk} - code: {name: fenics, url: 'https://bitbucket.org/fenics-project/dolfin/'} - cores: 24 - id_: 2b.1 - name: fenics_2b_ivan - timestamp: 2018/11/10 00:00:00 -- author: {email: ivan.yashchuk@aalto.fi, name: Ivan Yashchuk} - code: {name: fenics, url: 'https://bitbucket.org/fenics-project/dolfin/'} - cores: 24 - id_: 2c.1 - name: fenics_2c_ivan - timestamp: 2018/11/09 00:00:00 -- author: {email: ivan.yashchuk@vtt.fi, name: Ivan Yashchuk} - code: {name: fenics, url: 'https://bitbucket.org/fenics-project/dolfin/'} - cores: 24 - id_: 2d.1 - name: fenics_2d_ivan - timestamp: 2018/11/12 00:00:00 -- author: {email: trevor.keller@nist.gov, name: Trevor Keller} - code: {name: fipy, url: 'https://github.com/usnistgov/FiPy-spinodal-decomposition-benchmark/tree/master/periodic/'} - cores: 1 - id_: 1a.1 - name: fipy_1a_travis - timestamp: 2017/01/10 00:46:24 -- author: {email: trevor.keller@nist.gov, name: Trevor Keller} - code: {name: fipy, url: 'https://github.com/usnistgov/FiPy-spinodal-decomposition-benchmark/tree/master/no-flux/'} - cores: 1 - id_: 1b.1 - name: fipy_1b_travis - timestamp: 2017/01/10 01:11:25 -- author: {email: trevor.keller@nist.gov, name: Trevor Keller} - code: {name: fipy, url: 'https://github.com/usnistgov/FiPy-spinodal-decomposition-benchmark/tree/master/T-shape/'} - cores: 1 - id_: 1c.1 - name: fipy_1c_travis - timestamp: 2017/01/10 01:27:20 -- author: {email: daniel.wheeler2@gmail.com, name: Daniel Wheeler} - code: {name: fipy, url: 'https://gist.github.com/wd15/7e06a3141a6fbf317b1daf39ef1b0fbb'} - cores: 1 - id_: 3a.1 - name: fipy_3a - timestamp: 2018/02/02 00:00:00 -- author: {email: guyer@nist.gov, name: Jon Guyer} - code: {name: fipy, url: 'https://github.com/guyer/hackathonIII'} - cores: 1 - id_: 6a.0 - name: fipy_6a_macbookpro - timestamp: 2017/08/01 00:00:00 -- author: {email: guyer@nist.gov, name: Jon Guyer} - code: {name: fipy, url: 'https://github.com/guyer/hackathonIII'} - cores: 1 - id_: 6b.0 - name: fipy_6b_macbookpro - timestamp: 2017/08/01 00:00:00 -- author: {email: guyer@nist.gov, name: Jonathan Guyer} - code: {name: fipy, url: 'https://github.com/guyer/CHiMaDPhaseFieldVI'} - cores: 16 - id_: 7a.0 - name: fipy_7a_explicitDW - timestamp: 2018/03/07 00:00:00 -- author: {email: trevor.keller@nist.gov, name: Trevor Keller} - code: {name: custom, url: 'https://github.com/usnistgov/hiperc'} - cores: 1792 - id_: 1b.1 - name: hiperc_1b_cuda - timestamp: 2017/12/18 00:00:00 -- author: {email: trevor.keller@nist.gov, name: Trevor Keller} - code: {name: custom, url: 'https://github.com/usnistgov/hiperc'} - cores: 1792 - id_: 1b.1 - name: hiperc_1b_cuda_9pt - timestamp: 2017/12/18 00:00:00 -- author: {email: trevor.keller@nist.gov, name: Trevor Keller} - code: {name: hiperc, url: 'https://github.com/usnistgov/hiperc'} - cores: 1920 - id_: 7a.0 - name: hiperc_cuda - timestamp: 2018/02/19 00:00:00 -- author: {email: daniel.schwen@inl.gov, name: Daniel Schwen} - code: {name: moose, url: 'https://github.com/dschwen/CHiMaD_Hackathon'} - cores: 8 - id_: 1b.1 - name: inl_moose_1b - timestamp: 2017/01/24 00:00:00 -- author: {email: daniel.schwen@inl.gov, name: Daniel Schwen} - code: {name: moose, url: 'https://github.com/dschwen/CHiMaD_Hackathon'} - cores: 4 - id_: 1b.1 - name: inl_moose_1b_raspi - timestamp: 2017/01/27 00:00:00 -- author: {email: daniel.schwen@inl.gov, name: Daniel Schwen} - code: {name: moose, url: 'https://github.com/dschwen/CHiMaD_Hackathon'} - cores: 4 - id_: 1c.1 - name: inl_moose_1c - timestamp: 2017/06/01 00:00:00 -- author: {email: daniel.schwen@inl.gov, name: Daniel Schwen} - code: {name: moose, url: 'https://github.com/dschwen/moose/tree/benchmark'} - cores: 6 - id_: 3a.1 - name: inl_moose_3a - timestamp: 2018/10/02 00:00:00 -- author: {email: Larry.Aagesen@inl.gov, name: Larry Aagesen} - code: {name: moose, url: 'https://github.com/dschwen/CHiMaD_Hackathon'} - cores: 8 - id_: 6a.0 - name: inl_moose_6a - timestamp: 2017/07/26 00:00:00 -- author: {email: Larry.Aagesen@inl.gov, name: Larry Aagesen} - code: {name: moose, url: 'https://github.com/dschwen/CHiMaD_Hackathon'} - cores: 3 - id_: 6b.0 - name: inl_moose_6b - timestamp: 2017/07/27 00:00:00 -- author: {email: ikg5018@psu.edu, name: Ian Greenquist} - code: {name: moose, url: 'https://github.com/itgreenquist/moose/tree/benchmark'} - cores: 8 - id_: 1a.1 - name: itg_moose_1a - timestamp: 2018/03/29 00:00:00 -- author: {email: ikg5018@psu.edu, name: Ian Greenquist} - code: {name: moose, url: 'https://github.com/itgreenquist/moose/tree/benchmark'} - cores: 8 - id_: 1b.1 - name: itg_moose_1b - timestamp: 2018/03/19 00:00:00 -- author: {email: trevor.keller@nist.gov, name: Trevor Keller} - code: {name: mmsp, url: 'https://github.com/mesoscale/MMSP-spinodal-decomposition-benchmark/tree/master/periodic/'} - cores: 18 - id_: 1a.1 - name: mmsp_1a_raspi - timestamp: 2017/02/01 03:43:09 -- author: {email: trevor.keller@nist.gov, name: Trevor Keller} - code: {name: mmsp, url: 'https://github.com/mesoscale/MMSP-spinodal-decomposition-benchmark/tree/master/periodic/'} - cores: 2 - id_: 1a.1 - name: mmsp_1a_travis - timestamp: 2017/01/10 01:01:39 -- author: {email: trevor.keller@nist.gov, name: Trevor Keller} - code: {name: mmsp, url: 'https://github.com/mesoscale/MMSP-spinodal-decomposition-benchmark/tree/master/periodic/'} - cores: 12 - id_: 1a.1 - name: mmsp_1a_wrksttn - timestamp: 2017/02/10 13:34:42 -- author: {email: trevor.keller@nist.gov, name: Trevor Keller} - code: {name: mmsp, url: 'https://github.com/mesoscale/MMSP-spinodal-decomposition-benchmark/tree/master/no-flux/'} - cores: 18 - id_: 1b.1 - name: mmsp_1b_raspi - timestamp: 2017/02/04 12:48:15 -- author: {email: trevor.keller@nist.gov, name: Trevor Keller} - code: {name: mmsp, url: 'https://github.com/mesoscale/MMSP-spinodal-decomposition-benchmark/tree/master/no-flux/'} - cores: 2 - id_: 1b.1 - name: mmsp_1b_travis - timestamp: 2017/01/10 01:20:43 -- author: {email: trevor.keller@nist.gov, name: Trevor Keller} - code: {name: mmsp, url: 'https://github.com/mesoscale/MMSP-spinodal-decomposition-benchmark/tree/master/no-flux/'} - cores: 12 - id_: 1b.1 - name: mmsp_1b_wrksttn - timestamp: 2017/02/12 09:16:30 -- author: {email: trevor.keller@nist.gov, name: Trevor Keller} - code: {name: mmsp, url: 'https://github.com/mesoscale/MMSP-spinodal-decomposition-benchmark/tree/master/T-shape/'} - cores: 18 - id_: 1c.1 - name: mmsp_1c_raspi - timestamp: 2017/02/07 22:13:26 -- author: {email: trevor.keller@nist.gov, name: Trevor Keller} - code: {name: mmsp, url: 'https://github.com/mesoscale/MMSP-spinodal-decomposition-benchmark/tree/master/T-shape/'} - cores: 2 - id_: 1c.1 - name: mmsp_1c_travis - timestamp: 2017/01/10 01:40:01 -- author: {email: trevor.keller@nist.gov, name: Trevor Keller} - code: {name: mmsp, url: 'https://github.com/mesoscale/MMSP-spinodal-decomposition-benchmark/tree/master/T-shape/'} - cores: 12 - id_: 1c.1 - name: mmsp_1c_wrksttn - timestamp: 2017/02/14 08:42:23 -- author: {email: trevor.keller@nist.gov, name: Trevor Keller} - code: {name: mmsp, url: 'https://github.com/tkphd/CHiMaD-Hackathon/tree/master/Workshop5/electrochem'} - cores: 12 - id_: 6a.0 - name: mmsp_6a_wrksttn - timestamp: 2017/07/28 00:00:00 -- author: {email: andrea.jokisaari@northwestern.edu, name: A. M. Jokisaari} - code: {name: moose, url: 'https://bitbucket.org/ajokisaari/coral'} - cores: 6 - id_: 1a.1 - name: moose_1a_ia - timestamp: 2016/06/28 00:00:00 -- author: {email: jacob.hirschhorn@ufl.edu, name: Jacob Hirschhorn} - code: {name: moose, url: 'https://github.com/idaholab/moose'} - cores: 8 - id_: 1a.1 - name: moose_1a_jah - timestamp: 2018/08/22 00:00:00 -- author: {email: andrea.jokisaari@northwestern.edu, name: A. M. Jokisaari} - code: {name: moose, url: 'https://bitbucket.org/ajokisaari/coral'} - cores: 6 - id_: 1b.1 - name: moose_1b_ia - timestamp: 2016/06/28 00:00:00 -- author: {email: jacob.hirschhorn@ufl.edu, name: Jacob Hirschhorn} - code: {name: moose, url: 'https://github.com/idaholab/moose'} - cores: 8 - id_: 1b.1 - name: moose_1b_jah - timestamp: 2018/08/22 00:00:00 -- author: {email: andrea.jokisaari@northwestern.edu, name: A. M. Jokisaari} - code: {name: moose, url: 'https://bitbucket.org/ajokisaari/coral'} - cores: 6 - id_: 1b.1 - name: moose_1b_sta - timestamp: 2016/06/28 00:00:00 -- author: {email: andrea.jokisaari@northwestern.edu, name: A. M. Jokisaari} - code: {name: moose, url: 'https://bitbucket.org/ajokisaari/coral'} - cores: 6 - id_: 1c.1 - name: moose_1c_ia - timestamp: 2016/06/28 00:00:00 -- author: {email: jacob.hirschhorn@ufl.edu, name: Jacob Hirschhorn} - code: {name: moose, url: 'https://github.com/idaholab/moose'} - cores: 8 - id_: 1c.1 - name: moose_1c_jah - timestamp: 2018/08/22 00:00:00 -- author: {email: wuxueyang@ufl.edu, name: Xueyang Wu} - code: {name: moose, url: 'https://github.com/xueyang94/chimad1_spinodal_decomposition_c'} - cores: 8 - id_: 1c.1 - name: moose_1c_xueyang94 - timestamp: 2019/02/05 00:00:00 -- author: {email: andrea.jokisaari@northwestern.edu, name: A. M. Jokisaari} - code: {name: moose, url: 'https://bitbucket.org/ajokisaari/coral'} - cores: 20 - id_: 1d.1 - name: moose_1d_ia - timestamp: 2016/06/28 00:00:00 -- author: {email: jacob.hirschhorn@ufl.edu, name: Jacob Hirschhorn} - code: {name: moose, url: 'https://github.com/idaholab/moose'} - cores: 8 - id_: 1d.1 - name: moose_1d_jah - timestamp: 2018/08/23 00:00:00 -- author: {email: andrea.jokisaari@northwestern.edu, name: A. M. Jokisaari} - code: {name: moose, url: 'https://bitbucket.org/ajokisaari/coral'} - cores: 20 - id_: 1d.1 - name: moose_1d_sta - timestamp: 2016/06/28 00:00:00 -- author: {email: andrea.jokisaari@northwestern.edu, name: A. M. Jokisaari} - code: {name: moose, url: 'https://bitbucket.org/ajokisaari/coral'} - cores: 20 - id_: 2a.1 - name: moose_2a_ia - timestamp: 2016/06/28 00:00:00 -- author: {email: jacob.hirschhorn@ufl.edu, name: Jacob Hirschhorn} - code: {name: moose, url: 'https://github.com/idaholab/moose'} - cores: 8 - id_: 2a.1 - name: moose_2a_jah - timestamp: 2018/08/24 00:00:00 -- author: {email: andrea.jokisaari@northwestern.edu, name: A. M. Jokisaari} - code: {name: moose, url: 'https://bitbucket.org/ajokisaari/coral'} - cores: 20 - id_: 2a.1 - name: moose_2a_sta - timestamp: 2016/06/28 00:00:00 -- author: {email: wuxueyang@ufl.edu, name: Xueyang Wu} - code: {name: moose, url: 'https://github.com/xueyang94/chimad2_Oswald_Ripening_a'} - cores: 64 - id_: 2a.1 - name: moose_2a_xueyang94 - timestamp: 2019/02/11 00:00:00 -- author: {email: andrea.jokisaari@northwestern.edu, name: A. M. Jokisaari} - code: {name: moose, url: 'https://bitbucket.org/ajokisaari/coral'} - cores: 20 - id_: 2b.1 - name: moose_2b_ia - timestamp: 2016/06/28 00:00:00 -- author: {email: jacob.hirschhorn@ufl.edu, name: Jacob Hirschhorn} - code: {name: moose, url: 'https://github.com/idaholab/moose'} - cores: 8 - id_: 2b.1 - name: moose_2b_jah - timestamp: 2018/09/04 00:00:00 -- author: {email: andrea.jokisaari@northwestern.edu, name: A. M. Jokisaari} - code: {name: moose, url: 'https://bitbucket.org/ajokisaari/coral'} - cores: 20 - id_: 2b.1 - name: moose_2b_sta - timestamp: 2016/06/28 00:00:00 -- author: {email: andrea.jokisaari@northwestern.edu, name: A. M. Jokisaari} - code: {name: moose, url: 'https://bitbucket.org/ajokisaari/coral'} - cores: 6 - id_: 2c.1 - name: moose_2c_ia - timestamp: 2016/06/28 00:00:00 -- author: {email: jacob.hirschhorn@ufl.edu, name: Jacob Hirschhorn} - code: {name: moose, url: 'https://github.com/idaholab/moose'} - cores: 8 - id_: 2c.1 - name: moose_2c_jah - timestamp: 2018/09/11 00:00:00 -- author: {email: andrea.jokisaari@northwestern.edu, name: A. M. Jokisaari} - code: {name: moose, url: 'https://bitbucket.org/ajokisaari/coral'} - cores: 6 - id_: 2c.1 - name: moose_2c_sta - timestamp: 2016/06/28 00:00:00 -- author: {email: andrea.jokisaari@northwestern.edu, name: A. M. Jokisaari} - code: {name: moose, url: 'https://bitbucket.org/ajokisaari/coral'} - cores: 40 - id_: 2d.1 - name: moose_2d_ia - timestamp: 2016/06/28 00:00:00 -- author: {email: jacob.hirschhorn@ufl.edu, name: Jacob Hirschhorn} - code: {name: moose, url: 'https://github.com/idaholab/moose'} - cores: 8 - id_: 2d.1 - name: moose_2d_jah - timestamp: 2018/09/10 00:00:00 -- author: {email: andrea.jokisaari@northwestern.edu, name: A. M. Jokisaari} - code: {name: moose, url: 'https://bitbucket.org/ajokisaari/coral'} - cores: 40 - id_: 2d.1 - name: moose_2d_sta - timestamp: 2016/06/28 00:00:00 -- author: {email: fhilty@ufl.edu, name: Floyd Hilty} - code: {name: moose, url: 'https://github.com/FHilty/Benchmarks/blob/master/1a.i'} - cores: 8 - id_: 1a.1 - name: moose_fh_1a - timestamp: 2018/08/09 00:00:00 -- author: {email: fhilty@ufl.edu, name: Floyd Hilty} - code: {name: moose, url: 'https://github.com/FHilty/Benchmarks/blob/master/1b.i'} - cores: 8 - id_: 1b.1 - name: moose_fh_1b - timestamp: 2018/08/09 00:00:00 -- author: {email: fhilty@ufl.edu, name: Floyd Hilty} - code: {name: moose, url: 'https://github.com/FHilty/Benchmarks/blob/master/1c.i'} - cores: 8 - id_: 1c.1 - name: moose_fh_1c - timestamp: 2018/08/09 00:00:00 -- author: {email: fhilty@ufl.edu, name: Floyd Hilty} - code: {name: moose, url: 'https://github.com/FHilty/Benchmarks/blob/master/1d.i'} - cores: 8 - id_: 1d.1 - name: moose_fh_1d - timestamp: 2018/08/09 00:00:00 -- author: {email: fhilty@ufl.edu, name: Floyd Hilty} - code: {name: moose, url: 'https://github.com/FHilty/Benchmarks/blob/master/2a.i'} - cores: 8 - id_: 2a.1 - name: moose_fh_2a - timestamp: 2018/08/09 00:00:00 -- author: {email: fhilty@ufl.edu, name: Floyd Hilty} - code: {name: moose, url: 'https://github.com/FHilty/Benchmarks/blob/master/2b.i'} - cores: 8 - id_: 2b.1 - name: moose_fh_2b - timestamp: 2018/08/09 00:00:00 -- author: {email: fhilty@ufl.edu, name: Floyd Hilty} - code: {name: moose, url: 'https://github.com/FHilty/Benchmarks/blob/master/2c.i'} - cores: 8 - id_: 2c.1 - name: moose_fh_2c - timestamp: 2018/08/09 00:00:00 -- author: {email: fhilty@ufl.edu, name: Floyd Hilty} - code: {name: moose, url: 'https://github.com/FHilty/Benchmarks/blob/master/2d.i'} - cores: 8 - id_: 2d.1 - name: moose_fh_2d - timestamp: 2018/08/09 00:00:00 -- author: {email: ikg5018@psu.edu, name: Ian Greenquist} - code: {name: moose, url: 'https://github.com/itgreenquist/moose/tree/benchmark'} - cores: 8 - id_: 1c.1 - name: moose_ig_1c - timestamp: 2018/03/19 00:00:00 -- author: {email: ikg5018@psu.edu, name: Ian Greenquist} - code: {name: moose, url: 'https://github.com/itgreenquist/moose/tree/benchmark'} - cores: 8 - id_: 1d.1 - name: moose_itg_1d - timestamp: 2018/03/20 00:00:00 -- author: {email: ikg5018@psu.edu, name: Ian Greenquist} - code: {name: moose, url: 'https://github.com/itgreenquist/moose/tree/benchmark'} - cores: 8 - id_: 2a.1 - name: moose_itg_2a - timestamp: 2018/03/29 00:00:00 -- author: {email: ikg5018@psu.edu, name: Ian Greenquist} - code: {name: moose, url: 'https://github.com/itgreenquist/moose/tree/benchmark'} - cores: 8 - id_: 2b.1 - name: moose_itg_2b - timestamp: 2018/03/30 00:00:00 -- author: {email: ikg5018@psu.edu, name: Ian Greenquist} - code: {name: moose, url: 'https://github.com/itgreenquist/moose/tree/benchmark'} - cores: 8 - id_: 2c.1 - name: moose_itg_2c - timestamp: 2018/03/29 00:00:00 -- author: {email: ikg5018@psu.edu, name: Ian Greenquist} - code: {name: moose, url: 'https://github.com/itgreenquist/moose/tree/benchmark'} - cores: 8 - id_: 2d.1 - name: moose_itg_2d - timestamp: 2018/04/06 00:00:00 -- author: {email: ikg5018@psu.edu, name: Ian Greenquist} - code: {name: moose, url: 'https://github.com/itgreenquist/moose/tree/benchmark'} - cores: 8 - id_: 3a.1 - name: moose_itg_3a - timestamp: 2018/07/13 00:00:00 -- author: {email: pjs5523@psu.edu, name: PC. Simon} - code: {name: moose, url: 'https://github.com/simopier/CHiMaD/blob/master/H1.i'} - cores: 40 - id_: 1a.1 - name: moose_psu_1a_IA - timestamp: 2017/01/19 15:21:30 -- author: {email: pjs5523@psu.edu, name: PC. Simon} - code: {name: moose, url: 'https://github.com/simopier/CHiMaD/blob/master/H2.i'} - cores: 40 - id_: 1a.1 - name: moose_psu_1a_STA - timestamp: 2017/02/09 22:48:48 -- author: {email: pjs5523@psu.edu, name: PC. Simon} - code: {name: moose, url: 'https://github.com/simopier/CHiMaD/blob/master/H3.i'} - cores: 40 - id_: 1b.1 - name: moose_psu_1b_IA - timestamp: 2017/01/31 21:00:47 -- author: {email: pjs5523@psu.edu, name: PC. Simon} - code: {name: moose, url: 'https://github.com/simopier/CHiMaD/blob/master/H4.i'} - cores: 80 - id_: 1b.1 - name: moose_psu_1b_STA - timestamp: 2017/02/08 14:38:00 -- author: {email: pjs5523@psu.edu, name: PC. Simon} - code: {name: moose, url: 'https://github.com/simopier/CHiMaD/blob/master/H5.i'} - cores: 60 - id_: 1c.1 - name: moose_psu_1c_IA - timestamp: 2017/02/06 19:47:34 -- author: {email: pjs5523@psu.edu, name: PC. Simon} - code: {name: moose, url: 'https://github.com/simopier/CHiMaD/blob/master/H6.i'} - cores: 60 - id_: 1c.1 - name: moose_psu_1c_STA - timestamp: 2017/02/06 19:47:34 -- author: {email: pjs5523@psu.edu, name: PC. Simon} - code: {name: moose, url: 'https://github.com/simopier/CHiMaD/blob/master/H7.i'} - cores: 40 - id_: 1d.1 - name: moose_psu_1d_IA_2 - timestamp: 2017/01/31 21:01:55 -- author: {email: pjs5523@psu.edu, name: PC. Simon} - code: {name: moose, url: 'https://github.com/simopier/CHiMaD/blob/master/H8.i'} - cores: 100 - id_: 1d.1 - name: moose_psu_1d_STA - timestamp: 2017/02/07 10:36:59 -- author: {email: stvdwtt@umich.edu, name: Stephen DeWitt} - code: {name: prisms, url: 'https://github.com/prisms-center/phaseField/tree/master/applications/CHiMaD_benchmark1a'} - cores: 2 - id_: 1a.1 - name: prismspf_1a - timestamp: 2017/02/10 12:32:00 -- author: {email: stvdwtt@umich.edu, name: Stephen DeWitt} - code: {name: prisms, url: 'https://github.com/prisms-center/phaseField/tree/master/applications/CHiMaD_benchmark1a'} - cores: 2 - id_: 1b.1 - name: prismspf_1b - timestamp: 2017/02/10 12:32:00 -- author: {email: dmontiel@umich.edu, name: David Montiel} - code: {name: prisms, url: 'https://github.com/prisms-center/phaseField/tree/v2.0pre/applications/cahnHilliard_electrostatics'} - cores: 16 - id_: 6a.0 - name: prismspf_6a_flux_med - timestamp: 2017/07/28 00:00:00 -- author: {email: stvdwtt@umich.edu, name: Stephen DeWitt} - code: {name: prisms, url: 'https://github.com/prisms-center/phaseField/tree/v2.0pre/applications/cahnHilliard_electrostatics_Curved'} - cores: 16 - id_: 6b.0 - name: prismspf_6b - timestamp: 2017/07/28 00:00:00 -- author: {email: daniel.wheeler2@gmail.com, name: Daniel Wheeler} - code: {name: sfepy, url: 'https://github.com/wd15/sfepy-bm4/'} - cores: 1 - id_: 4a.1 - name: sfepy_4a_wd15 - timestamp: 2018/02/02 00:00:00 -- author: {email: daniel.wheeler2@gmail.com, name: Daniel Wheeler} - code: {name: sfepy, url: 'https://gist.github.com/wd15/7e06a3141a6fbf317b1daf39ef1b0fbb'} - cores: 1 - id_: 4b.1 - name: sfepy_4b_wd15_fake - timestamp: 2018/02/02 00:00:00 -- author: {email: daniel.wheeler2@gmail.com, name: Daniel Wheeler} - code: {name: sfepy, url: 'https://gist.github.com/wd15/7e06a3141a6fbf317b1daf39ef1b0fbb'} - cores: 1 - id_: 4c.1 - name: sfepy_4c_wd15_fake - timestamp: 2018/02/02 00:00:00 -- author: {email: daniel.wheeler2@gmail.com, name: Daniel Wheeler} - code: {name: sfepy, url: 'https://gist.github.com/wd15/7e06a3141a6fbf317b1daf39ef1b0fbb'} - cores: 1 - id_: 4d.1 - name: sfepy_4d_wd15_fake - timestamp: 2018/02/02 00:00:00 -- author: {email: daniel.wheeler2@gmail.com, name: Daniel Wheeler} - code: {name: sfepy, url: 'https://gist.github.com/wd15/7e06a3141a6fbf317b1daf39ef1b0fbb'} - cores: 1 - id_: 4e.1 - name: sfepy_4e_wd15_fake - timestamp: 2018/02/02 00:00:00 -- author: {email: daniel.wheeler2@gmail.com, name: Daniel Wheeler} - code: {name: sfepy, url: 'https://gist.github.com/wd15/7e06a3141a6fbf317b1daf39ef1b0fbb'} - cores: 1 - id_: 4f.1 - name: sfepy_4f_wd15_fake - timestamp: 2018/02/02 00:00:00 -- author: {email: daniel.wheeler2@gmail.com, name: Daniel Wheeler} - code: {name: sfepy, url: 'https://gist.github.com/wd15/7e06a3141a6fbf317b1daf39ef1b0fbb'} - cores: 1 - id_: 4g.1 - name: sfepy_4g_wd15_fake - timestamp: 2018/02/02 00:00:00 -- author: {email: daniel.wheeler2@gmail.com, name: Daniel Wheeler} - code: {name: sfepy, url: 'https://gist.github.com/wd15/7e06a3141a6fbf317b1daf39ef1b0fbb'} - cores: 1 - id_: 4h.1 - name: sfepy_4h_wd15_fake - timestamp: 2018/02/02 00:00:00 diff --git a/_data/migrate_yaml.py b/_data/migrate_yaml.py index c1653540c..fd80094b7 100644 --- a/_data/migrate_yaml.py +++ b/_data/migrate_yaml.py @@ -9,10 +9,31 @@ import glob # pylint: disable=redefined-builtin -from toolz.curried import pipe, valmap, itemmap, do, map +from toolz.curried import pipe, valmap, itemmap, do, get, map import yaml -from simulations import get_path, read_yaml + +def get_path(): + """Return the local file path for this file. + + Returns: + the filepath + """ + return pipe(__file__, os.path.realpath, os.path.split, get(0)) + + +def read_yaml(filepath): + """Read a YAML file + + Args: + filepath: the path to the YAML file + + Returns: + returns a dictionary + """ + with open(filepath) as stream: + data = yaml.safe_load(stream) + return data def migrate(func): diff --git a/_data/simulations.py b/_data/simulations.py deleted file mode 100644 index ec60b31f2..000000000 --- a/_data/simulations.py +++ /dev/null @@ -1,391 +0,0 @@ -"""Data pipelines to parse benchmark YAML for charts. - -Extracts benchmark YAML and combines with template chart YAML for Vega -charts. Run `python _data/charts.py` to build the charts. -""" -# pylint: disable=no-value-for-parameter - -import glob -import os -import json -import re - -from dateutil.parser import parse -import jinja2 - -# pylint: disable=redefined-builtin, no-name-in-module -from toolz.curried import map, pipe, get, curry, filter, compose, juxt -from toolz.curried import valmap, itemmap, groupby, memoize, keymap, update_in -from toolz.curried import valfilter -import yaml - - -def fcompose(*args): - """Helper function to compose functions. - - >>> f = lambda x: x - 2 - >>> g = lambda x: 2 * x - >>> f(g(3)) - 4 - >>> fcompose(g, f)(3) - 4 - - Args: - *args: tuple of functions - - Returns: - composed functions - """ - return compose(*args[::-1]) - - -# pylint: disable=invalid-name -sequence = fcompose - - -def read_yaml(filepath): - """Read a YAML file - - Args: - filepath: the path to the YAML file - - Returns: - returns a dictionary - """ - with open(filepath) as stream: - data = yaml.safe_load(stream) - return data - - -@curry -def write_json(data, filepath): - """Write a JSON file - - Args: - data: the dictionary to write - filepath: the path to the JSON file - - Returns: - returns a tuple of (filepath, data) - """ - with open(filepath, "w") as stream: - json.dump(data, stream, sort_keys=True, indent=2) - return (filepath, data) - - -def get_path(): - """Return the local file path for this file. - - Returns: - the filepath - """ - return pipe(__file__, os.path.realpath, os.path.split, get(0)) - - -@curry -def update_dict(dict_, **kwargs): - """Add keys to a new dictionary. - - Args: - dict_: the dictionary to add to - kwargs: the key, value pairs to add - - Returns: - a new dictionary - """ - return dict(list(dict_.items()) + list(kwargs.items())) - - -@curry -def filter_data(field, yaml_data): - """Extract a field of data from the YAML files. - - Args: - field: the name of the field to extract - yaml_data: the benchmark YAML data - - Returns: - the filtered data from the YAML data - """ - return pipe( - yaml_data, - dict, - valmap(lambda val: val["data"]), - valmap(filter(lambda item: item["name"].lower() == field)), - valmap(list), - valmap(get(0, default=None)), - valfilter(lambda x: x is not None), - itemmap(lambda item: (item[0], update_dict(item[1], name=item[0]))), - lambda dict_: sorted(list(dict_.values()), key=lambda item: item["name"]), - map( - update_in( - keys=["transform"], - func=lambda x: x + [dict(expr="datum.x > 0.01", type="filter")], - ) - ), - ) - - -@curry -def filter_memory_data(yaml_data): - """Filter the memory time data from the meta.yaml's - - Args: - yaml_data: the benchmark YAML data - - Returns: - memory versus time data - """ - - def time_ratio(data): - """Calcuate the sim_time over wall_time ration - """ - - def not0(value): - """Set to 1e-10 if 0 - """ - if value == 0: - return 1e-10 - return value - - return pipe( - data[-1], - juxt( - lambda x: x.get("sim_time", x.get("time")), - lambda x: x.get("wall_time", x.get("time")), - ), - lambda x: float(x[1]) / not0(float(x[0])), - ) - - def memory_usage(data): - """Calculate the memory usage in KB - """ - unit_map = dict(GB=1048576., KB=1., MB=1024., B=1. / 1024.) - if isinstance(data, dict): - data_ = data - else: - data_ = data[-1] - key = next(k for k in data_.keys() if "value" in k) - return float(data_[key]) * unit_map[data_.get("unit", "KB")] - - def make_datum(data): - """Build an item in the data list for one simulation - """ - return dict( - name="efficiency", - values=[ - dict( - time_ratio=time_ratio(data["run_time"]), - memory_usage=memory_usage(data["memory_usage"]), - ) - ], - ) - - return pipe( - yaml_data, - dict, - valmap(lambda x: x["data"]), - valmap( - filter(lambda item: item["name"].lower() in ("memory_usage", "run_time")) - ), - valmap(map(lambda x: (x["name"], x["values"]))), - valmap(dict), - valmap(make_datum), - itemmap(lambda item: (item[0], update_dict(item[1], name=item[0]))), - lambda dict_: sorted(list(dict_.values()), key=lambda item: item["name"]), - ) - - -def get_yaml_data(): - """Read in the YAML data but don't group - - Returns: - list of tuples of (name, data_dict) - """ - return pipe( - os.path.join(get_path(), "simulations/*/meta.yaml"), - glob.glob, - sorted, - map( - lambda path_: (os.path.split(os.path.split(path_)[0])[1], read_yaml(path_)) - ), - filter( - lambda item: item[0] not in ["example", "example_minimal", "test_lander"] - ), - ) - - -def vega2to3(data): - """Transform a Vega data list from version 2 to 3. - - Args: - data: vega data list - - Returns: - update vega data list - """ - - def keymapping(key): - """Map vega data keys from version 2 to 3 - - The mapping is `test` -> `expr` and `field` -> `as` otherwise - the input key is just returned. - - Args: - key: the key to map - - Returns: - a new key - """ - return dict(test="expr", field="as").get(key, key) - - update_transform = fcompose(map(keymap(keymapping)), list) - return pipe( - data, - map(update_in(keys=["transform"], func=update_transform, default=[])), - list, - ) - - -def get_data(filter_func): - """Read in the YAML data and group by benchmark id - - Args: - filter_func: function to filter data - - Returns: - a dictionary with benchmark ids as keys and lists of appropriate - data for values - """ - - return pipe( - get_yaml_data(), - groupby( - lambda item: "{0}.{1}".format( - item[1]["benchmark"]["id"], str(item[1]["benchmark"]["version"]) - ) - ), - valmap(filter_func), - valmap(vega2to3), - ) - - -def get_chart_file(j2_file_name): - """Get the name of the chart file - - Returns: - the chart YAML file - - """ - return os.path.join(get_path(), "charts", j2_file_name) - - -@curry -def write_chart_json(j2_file_name, item): - """Write a chart JSON file. - - Args: - j2_file_name: the name of the Jinja template file - item: a (benchmark_id, chart_dict) pair - - Returns: - returns the (filepath, json_data) pair - """ - file_name = fcompose( - lambda x: r"{0}_{1}".format(x, j2_file_name), - lambda x: re.sub(r"([0-9]+[abcd])\.(.+)\.yaml\.j2", r"\1\2.json", x), - ) - return pipe( - item[0], - file_name, - lambda file_: os.path.join(get_path(), "../_data/charts", file_), - write_json(item[1]), - ) - - -@memoize -def get_marks(): - """Get the mark data for the free energy charts - - Returns: - a dictionary defined in marks.yaml - """ - return pipe(os.path.join(get_path(), "marks.yaml"), read_yaml) - - -def process_chart(id_, data, j2_file_name): - """Process chart's YAML with data. - - Args: - id_: the benchmark ID - data: the data to process the YAML file - j2_file_name: the name of the j2 file to process - - Returns: - the rendered YAML as a dictionary - """ - return pipe( - get_chart_file(j2_file_name), - render_yaml( - data=data, id_=id_, marks=get_marks().get(id_, get_marks()["default"]) - ), - yaml.load, - ) - - -def to_datetime(datetime_str, format_="%Y/%m/%d %H:%M:%S"): - """Datetime formater for Jinja template. - """ - return parse(datetime_str).strftime(format_) - - -@curry -def render_yaml(tpl_path, **kwargs): - """Return the rendered yaml template. - - Args: - tpl_path: path to the YAML jinja template - **kwargs: data to render in the template - - Returns: - the rendered template string - """ - path, filename = os.path.split(tpl_path) - loader = jinja2.FileSystemLoader(path or "./") - env = jinja2.Environment(loader=loader) - env.filters["to_yaml"] = yaml.dump - env.filters["to_datetime"] = to_datetime - return env.get_template(filename).render(**kwargs) - - -def main(filter_func, j2_file_name): - """Generate the chart JSON files - - Args: - filter_func: function to filter simulaton YAML data - j2_file_name: the j2 file name to insert the data into - - Returns: - list of (filepath, chart_json) pairs - """ - return pipe( - get_data(filter_func), - valfilter(lambda x: len(x) > 0), - itemmap(lambda item: (item[0], process_chart(item[0], item[1], j2_file_name))), - itemmap(write_chart_json(j2_file_name)), - ) - - -def j2_to_json(path_in, path_out, **kwargs): - """Render a yaml.j2 chart to JSON. - - Args: - path_in: the j2 template path - path_out: the JSON path to write to - kwargs: data to pass to the j2 template - Returns: - the file path and JSON string - """ - return pipe( - render_yaml(path_in, **kwargs), yaml.load, write_json(filepath=path_out) - ) diff --git a/_data/table.yaml.j2 b/_data/table.yaml.j2 deleted file mode 100644 index 178627e23..000000000 --- a/_data/table.yaml.j2 +++ /dev/null @@ -1,14 +0,0 @@ ---- -data: -{%- for name, datum in data %} - - name: {{ name }} - code: - name: {{ datum.metadata.implementation.name }} - url: {{ datum.metadata.implementation.repo.url }} - id_: {{ datum.benchmark.id }}.{{ datum.benchmark.version }} - author: - email: {{ datum.metadata.author.email }} - name: {{ datum.metadata.author.first }} {{ datum.metadata.author.last }} - timestamp: {{ datum.metadata.timestamp | to_datetime(format_="%Y/%m/%d %H:%M:%S") }} - cores: {{ datum.metadata.hardware.cores }} -{%- endfor %} diff --git a/_includes/benchmarks.html b/_includes/benchmarks.html index 8249354a1..2a17119c3 100644 --- a/_includes/benchmarks.html +++ b/_includes/benchmarks.html @@ -18,8 +18,7 @@