From a83eb0090239036b4646f34bb1178c6d22b77251 Mon Sep 17 00:00:00 2001 From: Carol Jiang Date: Fri, 22 Dec 2023 15:26:17 -0500 Subject: [PATCH] updated readthedocs --- brainscore_language/document_plugins.py | 196 ++++++ docs2/document_plugins.py | 157 +++++ docs2/source/_static/overrides.css | 5 + docs2/source/bibtex/data.bib | 40 ++ docs2/source/bibtex/refs.bib | 40 ++ docs2/source/conf.py | 5 +- .../brainscore_language.document_plugins.rst | 38 ++ .../brainscore_language.parser.rst | 29 + .../_autosummary/brainscore_language.rst | 1 + docs2/source/modules/plugins.rst | 578 ++++++++++++++++-- 10 files changed, 1042 insertions(+), 47 deletions(-) create mode 100644 brainscore_language/document_plugins.py create mode 100644 docs2/document_plugins.py create mode 100644 docs2/source/_static/overrides.css create mode 100644 docs2/source/bibtex/data.bib create mode 100644 docs2/source/bibtex/refs.bib create mode 100644 docs2/source/modules/_autosummary/brainscore_language.document_plugins.rst create mode 100644 docs2/source/modules/_autosummary/brainscore_language.parser.rst diff --git a/brainscore_language/document_plugins.py b/brainscore_language/document_plugins.py new file mode 100644 index 00000000..3cc13f1b --- /dev/null +++ b/brainscore_language/document_plugins.py @@ -0,0 +1,196 @@ +""" Make plugin details available to readthedocs """ + +import json +import logging +from pathlib import Path +import re +from rstcloth import RstCloth +from typing import Dict, List, Union + +from brainscore_language import load_dataset, load_metric + +# BIBS_DIR = 'docs/source/bibtex/' +BIBS_DIR = '../docs2/source/bibtex/' +GITHUB_DIR = 'https://github.com/brain-score/language/tree/main/brainscore_language/' +# PLUGINS_DOC_FP = 'docs/source/modules/plugins.rst' +PLUGINS_DOC_FP = '../docs2/source/modules/plugins.rst' +PLUGINS_LIST_FP = 'brainscore_language/plugin_management/all_plugins.json' +PLUGIN_DIRS = ['benchmarks', 'data', 'metrics', 'models'] +PLUGIN_TYPE_MAP = {plugin_dirtype:plugin_dirtype.strip('s') for plugin_dirtype in PLUGIN_DIRS} + + +def _get_module_plugin_names(plugin_type:str, plugin_dir:Path) -> List[str]: + """ Returns list of plugins registered by module """ + init_fp = plugin_dir / "__init__.py" + registry = PLUGIN_TYPE_MAP[plugin_type] + "_registry" + + with open(init_fp, 'r') as f: + text = f.read() + registered_plugins = re.findall(registry+'\[(.*)\]', text) + cleaned_plugin_names = [name.replace('"', '').replace('\'', '') for name in registered_plugins] + + return cleaned_plugin_names + +def _id_from_bibtex(bibtex:str) -> str: + """ Returns BibTeX identifier from BibTeX """ + return re.search('\{(.*?),', bibtex).group(1) + +def _data_metric_from_benchmark(filepath): + data_pattern = 'self.data = load_data' + metric_pattern = 'self.metric = load_metric' + with open(filepath, 'r') as f: + content = f.read() + + data_match = re.search(r'load_dataset\([\'"]([^\'"]*)[\'"]\)', content) + metric_match = re.search(r'load_metric\([\'"]([^\'"]*)[\'"]\)', content) + + data_value = data_match.group(1) if data_match else None + metric_value = metric_match.group(1) if metric_match else None + + return data_value, metric_value + +def get_all_plugin_info() -> Dict[str, Dict[str, Dict[str, Union[list, str, str]]]]: + """Add all plugins to respective type registries + + Returns a dict where key is plugin type, + value is a dict where key is name of plugin dir, + value is a dict of plugin info: + + plugin_names: list of names of all plugins registered by module + bibtex: a BibTeX string + bibtex_id: BibTeX identifier + """ + all_plugin_info = {} + for plugin_type in PLUGIN_DIRS: + plugins_dir = Path(Path(__file__).parent, plugin_type) + for plugin_dir in plugins_dir.glob('[!._]*'): + plugin_dirname = str(plugin_dir) + + if plugin_type not in all_plugin_info: + all_plugin_info[plugin_type] = {plugin_dirname:{}} + else: + all_plugin_info[plugin_type].update({plugin_dirname:{}}) + + plugin_dir_dict = all_plugin_info[plugin_type][plugin_dirname] + + plugin_names = _get_module_plugin_names(plugin_type, plugin_dir) + plugin_module_path = plugin_dirname.replace('/', '.') + plugin_module = __import__(plugin_module_path, fromlist=['BIBTEX']) + + plugin_dir_dict['plugin_names'] = plugin_names + if hasattr(plugin_module, 'BIBTEX'): + plugin_dir_dict['bibtex'] = plugin_module.BIBTEX + plugin_dir_dict['bibtex_id'] = _id_from_bibtex(plugin_module.BIBTEX) + + if plugin_type == 'benchmarks': + data_value, metric_value = _data_metric_from_benchmark(plugin_dirname + '/benchmark.py') + if data_value: + plugin_dir_dict['data_value'] = data_value + if metric_value: + plugin_dir_dict['metric_value'] = metric_value + + return all_plugin_info + +def _remove_duplicate_bibs(plugins_with_bibtex=Dict[str, Dict]): + """ Returns list of unique BibTeX to add """ + bibtex_data = {v['bibtex_id']:v['bibtex'] for v in plugins_with_bibtex.values()} + alphabetized_bibtex = dict(sorted(bibtex_data.items())) + deduped_bibtex = list(alphabetized_bibtex.values()) + + return deduped_bibtex + +def _record_bibtex(bibtex_to_add:List[str], plugins_bib_fp:str): + """ insert new BibTeX into respective .bib files """ + if not Path(BIBS_DIR).exists(): + Path(BIBS_DIR).mkdir(parents=True) + with open(plugins_bib_fp, "w+") as f: + for bibtex in bibtex_to_add: + f.write(bibtex) + f.write('\n') + +def create_bibfile(plugins=Dict[str, Dict], plugin_type='refs'): + """ For all plugins, add bibtex (if present) to .bib files """ + if plugin_type == 'refs': + plugins = dict(ele for sub in plugins.values() for ele in sub.items()) + # drop plugins without bibtex + plugins_with_bibtex = {k:v for k,v in plugins.items() if 'bibtex' in v.keys()} + if len(plugins_with_bibtex.keys()) > 0: + plugins_bib_fp = Path(BIBS_DIR + plugin_type + '.bib') + # add bibtex (if present) to .bib files + bibtex_to_add = _remove_duplicate_bibs(plugins_with_bibtex) + _record_bibtex(bibtex_to_add, plugins_bib_fp) + +def _prepare_content(all_plugin_info:Dict[str, Dict]) -> Dict[str, Dict]: + """Converts plugin information into rst format + + Returns a dict where key is plugin type, value is a dict + of plugin names (str) mapped to a dict of their info + + NOTE: info is currently plugin directory paths and BiBTeX citations, + but could expand to e.g. include description of plugin + """ + prepared_plugin_info = {} + for plugin_type in all_plugin_info: + plugin_type_title = plugin_type.capitalize() + prepared_plugin_info[plugin_type_title] = {name:{'dirname':k, + 'citation':(':cite:label:`' + v['bibtex_id'] +'`' + if 'bibtex_id' in v.keys() else None), + 'data':(f":ref:`{v['data_value']} <{v['data_value']}>`" + if 'data_value' in v.keys() else None), + 'metric':(f":ref:`{v['metric_value']} <{v['metric_value']}>`" + if 'metric_value' in v.keys() else None)} + for k,v in all_plugin_info[plugin_type].items() + for name in v['plugin_names']} + return prepared_plugin_info + +def _write_to_rst(plugin_info:Dict[str,Dict]): + """ Writes plugin info to readthedocs plugins.rst """ + upper_levels = set() + with open(PLUGINS_DOC_FP, 'w+') as f: + doc = RstCloth(f) + doc.ref_target(name="plugins") + doc.newline() + doc.title('Plugins') + doc.newline() + for plugin_type in plugin_info: + doc.newline() + doc.h3(plugin_type) + doc.content(f'The following {plugin_type} are available. Plugin identifiers are grouped by directory, shown in bold.') + doc.newline() + for plugin in plugin_info[plugin_type]: + location = plugin_info[plugin_type][plugin]['dirname'] + location_link = f'`{location} <{GITHUB_DIR}{location}>`_' + if location not in upper_levels: + idx = location.index('/') + doc.h4(location[idx+1:].capitalize()) + doc.content(f'Location: {location_link}') + doc.newline() + upper_levels.add(location) + doc.newline() + doc.ref_target(plugin) + doc.h6(plugin) + doc.newline() + if plugin_info[plugin_type][plugin]['data']: + doc.content(f"Data: {plugin_info[plugin_type][plugin]['data']}", indent=2) + doc.newline() + if plugin_info[plugin_type][plugin]['metric']: + doc.content(f"Metric: {plugin_info[plugin_type][plugin]['metric']}", indent=2) + doc.newline() + if plugin_info[plugin_type][plugin]['citation']: + doc.content(f"Citation: {plugin_info[plugin_type][plugin]['citation']}", indent=2) + doc.newline() + doc.newline() + doc.h2('Bibliography') + doc.directive(name="bibliography", fields=[('all','')]) + +def update_readthedocs(all_plugin_info:Dict[str,Dict]): + """ For all plugins, add name and info to readthedocs (plugins.rst) """ + prepared_plugin_info = _prepare_content(all_plugin_info) # rst formatting + _write_to_rst(prepared_plugin_info) + +if __name__ == '__main__': + all_plugin_info = get_all_plugin_info() + for plugin_type in all_plugin_info: + create_bibfile(all_plugin_info[plugin_type], plugin_type) # plugin type .bib file + create_bibfile(all_plugin_info) # one .bib file to rule them all + update_readthedocs(all_plugin_info) diff --git a/docs2/document_plugins.py b/docs2/document_plugins.py new file mode 100644 index 00000000..795e50e5 --- /dev/null +++ b/docs2/document_plugins.py @@ -0,0 +1,157 @@ +""" Make plugin details available to readthedocs """ + +import json +import logging +from pathlib import Path +import re +from rstcloth import RstCloth +from typing import Dict, List, Union + +# BIBS_DIR = 'docs/source/bibtex/' +BIBS_DIR = 'source/bibtex/' +# PLUGINS_DOC_FP = 'docs/source/modules/plugins.rst' +PLUGINS_DOC_FP = 'source/modules/plugins2.rst' +PLUGINS_LIST_FP = '../brainscore_language/plugin_management/all_plugins.json' +PLUGIN_DIRS = ['benchmarks', 'data', 'metrics', 'models'] +PLUGIN_TYPE_MAP = {plugin_dirtype:plugin_dirtype.strip('s') for plugin_dirtype in PLUGIN_DIRS} + + +def _get_module_plugin_names(plugin_type:str, plugin_dir:Path) -> List[str]: + """ Returns list of plugins registered by module """ + init_fp = plugin_dir / "__init__.py" + registry = PLUGIN_TYPE_MAP[plugin_type] + "_registry" + + with open(init_fp, 'r') as f: + text = f.read() + registered_plugins = re.findall(registry+'\[(.*)\]', text) + cleaned_plugin_names = [name.replace('"', '').replace('\'', '') for name in registered_plugins] + + return cleaned_plugin_names + +def _id_from_bibtex(bibtex:str) -> str: + """ Returns BibTeX identifier from BibTeX """ + return re.search('\{(.*?),', bibtex).group(1) + +def get_all_plugin_info() -> Dict[str, Dict[str, Dict[str, Union[list, str, str]]]]: + """Add all plugins to respective type registries + + Returns a dict where key is plugin type, + value is a dict where key is name of plugin dir, + value is a dict of plugin info: + + plugin_names: list of names of all plugins registered by module + bibtex: a BibTeX string + bibtex_id: BibTeX identifier + """ + all_plugin_info = {} + for plugin_type in PLUGIN_DIRS: + # plugins_dir = Path(Path(__file__).parents[1], plugin_type) + plugins_dir = Path(Path(__file__).resolve().parents[1], 'brainscore_language', plugin_type) + print('plugins_dir', plugins_dir.resolve()) + for plugin_dir in plugins_dir.glob('[!._]*'): + idx = str(plugin_dir).index('brainscore_language') + # plugin_dirname = str(plugin_dir) + plugin_dirname = str(plugin_dir)[idx:] + # print(plugin_dirname) + + if plugin_type not in all_plugin_info: + all_plugin_info[plugin_type] = {plugin_dirname:{}} + else: + all_plugin_info[plugin_type].update({plugin_dirname:{}}) + + plugin_dir_dict = all_plugin_info[plugin_type][plugin_dirname] + + plugin_names = _get_module_plugin_names(plugin_type, plugin_dir) + plugin_module_path = plugin_dirname.replace('/', '.') + plugin_module = __import__(plugin_dirname, fromlist=['BIBTEX']) + # print('plugin_module', plugin_module) + + plugin_dir_dict['plugin_names'] = plugin_names + if hasattr(plugin_module, 'BIBTEX'): + plugin_dir_dict['bibtex'] = plugin_module.BIBTEX + plugin_dir_dict['bibtex_id'] = _id_from_bibtex(plugin_module.BIBTEX) + + return all_plugin_info + +def _remove_duplicate_bibs(plugins_with_bibtex=Dict[str, Dict]): + """ Returns list of unique BibTeX to add """ + bibtex_data = {v['bibtex_id']:v['bibtex'] for v in plugins_with_bibtex.values()} + alphabetized_bibtex = dict(sorted(bibtex_data.items())) + deduped_bibtex = list(alphabetized_bibtex.values()) + + return deduped_bibtex + +def _record_bibtex(bibtex_to_add:List[str], plugins_bib_fp:str): + """ insert new BibTeX into respective .bib files """ + print('recording bibtex', Path(BIBS_DIR).resolve()) + if not Path(BIBS_DIR).exists(): + Path(BIBS_DIR).mkdir(parents=True) + print('plugins_bib_fp', plugins_bib_fp) + with open(plugins_bib_fp, "w+") as f: + for bibtex in bibtex_to_add: + f.write(bibtex) + f.write('\n') + +def create_bibfile(plugins=Dict[str, Dict], plugin_type='refs'): + """ For all plugins, add bibtex (if present) to .bib files """ + if plugin_type == 'refs': + plugins = dict(ele for sub in plugins.values() for ele in sub.items()) + # drop plugins without bibtex + plugins_with_bibtex = {k:v for k,v in plugins.items() if 'bibtex' in v.keys()} + if len(plugins_with_bibtex.keys()) > 0: + plugins_bib_fp = Path(BIBS_DIR + plugin_type + '.bib') + # add bibtex (if present) to .bib files + bibtex_to_add = _remove_duplicate_bibs(plugins_with_bibtex) + _record_bibtex(bibtex_to_add, plugins_bib_fp) + +def _prepare_content(all_plugin_info:Dict[str, Dict]) -> Dict[str, Dict]: + """Converts plugin information into rst format + + Returns a dict where key is plugin type, value is a dict + of plugin names (str) mapped to a dict of their info + + NOTE: info is currently plugin directory paths and BiBTeX citations, + but could expand to e.g. include description of plugin + """ + prepared_plugin_info = {} + for plugin_type in all_plugin_info: + plugin_type_title = plugin_type.capitalize() + prepared_plugin_info[plugin_type_title] = {name:{'dirname':k, + 'citation':(':cite:label:`' + v['bibtex_id'] +'`' + if 'bibtex_id' in v.keys() else None)} + for k,v in all_plugin_info[plugin_type].items() + for name in v['plugin_names']} + return prepared_plugin_info + +def _write_to_rst(plugin_info:Dict[str,Dict]): + """ Writes plugin info to readthedocs plugins.rst """ + print(PLUGINS_DOC_FP) + with open(PLUGINS_DOC_FP, 'w+') as f: + doc = RstCloth(f) + doc.ref_target(name="plugins") + doc.newline() + doc.title('Plugins') + doc.newline() + for plugin_type in plugin_info: + doc.h3(plugin_type) + for plugin in plugin_info[plugin_type]: + doc.h4(plugin) + doc.content(plugin_info[plugin_type][plugin]['dirname']) + doc.newline() + if plugin_info[plugin_type][plugin]['citation']: + doc.content(plugin_info[plugin_type][plugin]['citation']) + doc.newline() + doc.h2('Bibliography') + doc.directive(name="bibliography", fields=[('all','')]) + +def update_readthedocs(all_plugin_info:Dict[str,Dict]): + """ For all plugins, add name and info to readthedocs (plugins.rst) """ + prepared_plugin_info = _prepare_content(all_plugin_info) # rst formatting + _write_to_rst(prepared_plugin_info) + +if __name__ == '__main__': + all_plugin_info = get_all_plugin_info() + for plugin_type in all_plugin_info: + create_bibfile(all_plugin_info[plugin_type], plugin_type) # plugin type .bib file + create_bibfile(all_plugin_info) # one .bib file to rule them all + update_readthedocs(all_plugin_info) diff --git a/docs2/source/_static/overrides.css b/docs2/source/_static/overrides.css new file mode 100644 index 00000000..b67302d2 --- /dev/null +++ b/docs2/source/_static/overrides.css @@ -0,0 +1,5 @@ +@import url("sphinx_rtd_theme.css"); + +h6 { + font-size: 5px; +} \ No newline at end of file diff --git a/docs2/source/bibtex/data.bib b/docs2/source/bibtex/data.bib new file mode 100644 index 00000000..589d0bf9 --- /dev/null +++ b/docs2/source/bibtex/data.bib @@ -0,0 +1,40 @@ +@article{blank2014functional, + title={A functional dissociation between language and multiple-demand systems revealed in patterns of BOLD signal fluctuations}, + author={Blank, Idan and Kanwisher, Nancy and Fedorenko, Evelina}, + journal={Journal of neurophysiology}, + volume={112}, + number={5}, + pages={1105--1118}, + year={2014}, + publisher={American Physiological Society Bethesda, MD} +} +@article{fedorenko2016neural, + title={Neural correlate of the construction of sentence meaning}, + author={Fedorenko, Evelina and Scott, Terri L and Brunner, Peter and Coon, William G and Pritchett, Brianna and + Schalk, Gerwin and Kanwisher, Nancy}, + journal={Proceedings of the National Academy of Sciences}, + volume={113}, + number={41}, + pages={E6256--E6262}, + year={2016}, + publisher={National Acad Sciences} +} +@proceedings{futrell2018natural, + title={The Natural Stories Corpus}, + author={Futrell, Richard and Gibson, Edward and Tily, Harry J. and Blank, Idan and Vishnevetsky, Anastasia and + Piantadosi, Steven T. and Fedorenko, Evelina}, + conference={International Conference on Language Resources and Evaluation (LREC)}, + url={http://www.lrec-conf.org/proceedings/lrec2018/pdf/337.pdf}, + year={2018} +} +@article{pereira2018toward, + title={Toward a universal decoder of linguistic meaning from brain activation}, + author={Pereira, Francisco and Lou, Bin and Pritchett, Brianna and Ritter, Samuel and Gershman, Samuel J + and Kanwisher, Nancy and Botvinick, Matthew and Fedorenko, Evelina}, + journal={Nature communications}, + volume={9}, + number={1}, + pages={1--13}, + year={2018}, + publisher={Nature Publishing Group} +} diff --git a/docs2/source/bibtex/refs.bib b/docs2/source/bibtex/refs.bib new file mode 100644 index 00000000..589d0bf9 --- /dev/null +++ b/docs2/source/bibtex/refs.bib @@ -0,0 +1,40 @@ +@article{blank2014functional, + title={A functional dissociation between language and multiple-demand systems revealed in patterns of BOLD signal fluctuations}, + author={Blank, Idan and Kanwisher, Nancy and Fedorenko, Evelina}, + journal={Journal of neurophysiology}, + volume={112}, + number={5}, + pages={1105--1118}, + year={2014}, + publisher={American Physiological Society Bethesda, MD} +} +@article{fedorenko2016neural, + title={Neural correlate of the construction of sentence meaning}, + author={Fedorenko, Evelina and Scott, Terri L and Brunner, Peter and Coon, William G and Pritchett, Brianna and + Schalk, Gerwin and Kanwisher, Nancy}, + journal={Proceedings of the National Academy of Sciences}, + volume={113}, + number={41}, + pages={E6256--E6262}, + year={2016}, + publisher={National Acad Sciences} +} +@proceedings{futrell2018natural, + title={The Natural Stories Corpus}, + author={Futrell, Richard and Gibson, Edward and Tily, Harry J. and Blank, Idan and Vishnevetsky, Anastasia and + Piantadosi, Steven T. and Fedorenko, Evelina}, + conference={International Conference on Language Resources and Evaluation (LREC)}, + url={http://www.lrec-conf.org/proceedings/lrec2018/pdf/337.pdf}, + year={2018} +} +@article{pereira2018toward, + title={Toward a universal decoder of linguistic meaning from brain activation}, + author={Pereira, Francisco and Lou, Bin and Pritchett, Brianna and Ritter, Samuel and Gershman, Samuel J + and Kanwisher, Nancy and Botvinick, Matthew and Fedorenko, Evelina}, + journal={Nature communications}, + volume={9}, + number={1}, + pages={1--13}, + year={2018}, + publisher={Nature Publishing Group} +} diff --git a/docs2/source/conf.py b/docs2/source/conf.py index f6e384e5..ddc40a5b 100644 --- a/docs2/source/conf.py +++ b/docs2/source/conf.py @@ -35,7 +35,9 @@ 'recommonmark', 'sphinx.ext.viewcode', 'sphinx_rtd_theme', - 'sphinxcontrib.bibtex' + 'sphinxcontrib.bibtex', + 'rstcloth', + 'sphinx.ext.autosectionlabel' ] autosummary_generate = True # Turn on sphinx.ext.autosummary @@ -64,6 +66,7 @@ # a list of builtin themes. # html_theme = "sphinx_rtd_theme" +html_style = "overrides.css" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/docs2/source/modules/_autosummary/brainscore_language.document_plugins.rst b/docs2/source/modules/_autosummary/brainscore_language.document_plugins.rst new file mode 100644 index 00000000..4311171c --- /dev/null +++ b/docs2/source/modules/_autosummary/brainscore_language.document_plugins.rst @@ -0,0 +1,38 @@ +brainscore\_language.document\_plugins +====================================== + +.. automodule:: brainscore_language.document_plugins + + + + + + + + .. rubric:: Functions + + .. autosummary:: + + _data_metric_from_benchmark + _get_module_plugin_names + _id_from_bibtex + _prepare_content + _record_bibtex + _remove_duplicate_bibs + _write_to_rst + create_bibfile + get_all_plugin_info + update_readthedocs + + + + + + + + + + + + + diff --git a/docs2/source/modules/_autosummary/brainscore_language.parser.rst b/docs2/source/modules/_autosummary/brainscore_language.parser.rst new file mode 100644 index 00000000..5f95208e --- /dev/null +++ b/docs2/source/modules/_autosummary/brainscore_language.parser.rst @@ -0,0 +1,29 @@ +brainscore\_language.parser +=========================== + +.. automodule:: brainscore_language.parser + + + + + + + + .. rubric:: Functions + + .. autosummary:: + + parse + + + + + + + + + + + + + diff --git a/docs2/source/modules/_autosummary/brainscore_language.rst b/docs2/source/modules/_autosummary/brainscore_language.rst index e553f190..383aba0e 100644 --- a/docs2/source/modules/_autosummary/brainscore_language.rst +++ b/docs2/source/modules/_autosummary/brainscore_language.rst @@ -49,6 +49,7 @@ brainscore_language.artificial_subject brainscore_language.benchmark_helpers + brainscore_language.document_plugins brainscore_language.model_helpers brainscore_language.submission brainscore_language.utils diff --git a/docs2/source/modules/plugins.rst b/docs2/source/modules/plugins.rst index 9be29948..b6a6bb8a 100644 --- a/docs2/source/modules/plugins.rst +++ b/docs2/source/modules/plugins.rst @@ -1,103 +1,589 @@ - .. _plugins: + ======= Plugins ======= + Benchmarks ~~~~~~~~~~ +The following Benchmarks are available. Plugin identifiers are grouped +by directory, shown in bold. + +Futrell2018 ++++++++++++ +Location: `benchmarks/futrell2018 +`_ + + +.. _Futrell2018-pearsonr: +Futrell2018-pearsonr +;;;;;;;;;;;;;;;;;;;; + + Data: :ref:`Futrell2018 ` + + Metric: :ref:`pearsonr ` + + +Pereira2018 ++++++++++++ +Location: `benchmarks/pereira2018 +`_ + + +.. _Pereira2018.243sentences-linear: Pereira2018.243sentences-linear -+++++++++++++++++++++++++++++++ -brainscore_language/benchmarks/pereira2018 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -:cite:label:`pereira2018toward` + Data: :ref:`Pereira2018.language ` + Metric: :ref:`linear_pearsonr ` + + + +.. _Pereira2018.384sentences-linear: Pereira2018.384sentences-linear -+++++++++++++++++++++++++++++++ -brainscore_language/benchmarks/pereira2018 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -:cite:label:`pereira2018toward` + Data: :ref:`Pereira2018.language ` -Futrell2018-pearsonr -++++++++++++++++++++ -brainscore_language/benchmarks/futrell2018 + Metric: :ref:`linear_pearsonr ` +Wikitext_next_word +++++++++++++++++++ +Location: `benchmarks/wikitext_next_word +`_ + + +.. _Wikitext-accuracy: Wikitext-accuracy -+++++++++++++++++ -brainscore_language/benchmarks/wikitext_next_word +;;;;;;;;;;;;;;;;; + + Data: :ref:`wikitext-2/test ` + + Metric: :ref:`accuracy ` + + +Blank2014 ++++++++++ +Location: `benchmarks/blank2014 +`_ + + +.. _Blank2014-linear: +Blank2014-linear +;;;;;;;;;;;;;;;; + + Data: :ref:`Blank2014.fROI ` + + Metric: :ref:`linear_pearsonr ` + + +Syntaxgym ++++++++++ +Location: `benchmarks/syntaxgym +`_ + + +.. _syntaxgym-center_embed: +syntaxgym-center_embed +;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-center_embed_mod: +syntaxgym-center_embed_mod +;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-cleft: +syntaxgym-cleft +;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-cleft_modifier: +syntaxgym-cleft_modifier +;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-fgd_hierarchy: +syntaxgym-fgd_hierarchy +;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-fgd_object: +syntaxgym-fgd_object +;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-fgd_pp: +syntaxgym-fgd_pp +;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-fgd_subject: +syntaxgym-fgd_subject +;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-mvrr: +syntaxgym-mvrr +;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-mvrr_mod: +syntaxgym-mvrr_mod +;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-npi_orc_any: +syntaxgym-npi_orc_any +;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-npi_orc_ever: +syntaxgym-npi_orc_ever +;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-npi_src_any: +syntaxgym-npi_src_any +;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-npi_src_ever: +syntaxgym-npi_src_ever +;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-npz_ambig: +syntaxgym-npz_ambig +;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-npz_ambig_mod: +syntaxgym-npz_ambig_mod +;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-npz_obj: +syntaxgym-npz_obj +;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-npz_obj_mod: +syntaxgym-npz_obj_mod +;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-number_orc: +syntaxgym-number_orc +;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-number_prep: +syntaxgym-number_prep +;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-number_src: +syntaxgym-number_src +;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-reflexive_orc_fem: +syntaxgym-reflexive_orc_fem +;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-reflexive_orc_masc: +syntaxgym-reflexive_orc_masc +;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-reflexive_prep_fem: +syntaxgym-reflexive_prep_fem +;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-reflexive_prep_masc: +syntaxgym-reflexive_prep_masc +;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-reflexive_src_fem: +syntaxgym-reflexive_src_fem +;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-reflexive_src_masc: +syntaxgym-reflexive_src_masc +;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-subordination: +syntaxgym-subordination +;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-subordination_orc-orc: +syntaxgym-subordination_orc-orc +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-subordination_pp-pp: +syntaxgym-subordination_pp-pp +;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-subordination_src-src: +syntaxgym-subordination_src-src +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + -:cite:label:`merity2017pointer` Data ~~~~ +The following Data are available. Plugin identifiers are grouped by +directory, shown in bold. + +Blank2014 ++++++++++ +Location: `data/blank2014 +`_ + + +.. _Blank2014.fROI: Blank2014.fROI -++++++++++++++ -brainscore_language/data/blank2014 +;;;;;;;;;;;;;; -:cite:label:`blank2014functional` + Citation: :cite:label:`blank2014functional` -Pereira2018.language -++++++++++++++++++++ -brainscore_language/data/pereira2018 -:cite:label:`pereira2018toward` +Fedorenko2016 ++++++++++++++ +Location: `data/fedorenko2016 +`_ -Pereira2018.auditory -++++++++++++++++++++ -brainscore_language/data/pereira2018 -:cite:label:`pereira2018toward` +.. _Fedorenko2016.language: +Fedorenko2016.language +;;;;;;;;;;;;;;;;;;;;;; + + Citation: :cite:label:`fedorenko2016neural` + Futrell2018 +++++++++++ -brainscore_language/data/futrell2018 +Location: `data/futrell2018 +`_ -:cite:label:`futrell2018natural` -Fedorenko2016.language -++++++++++++++++++++++ -brainscore_language/data/fedorenko2016 +.. _Futrell2018: +Futrell2018 +;;;;;;;;;;; + + Citation: :cite:label:`futrell2018natural` -:cite:label:`fedorenko2016neural` +Pereira2018 ++++++++++++ +Location: `data/pereira2018 +`_ + + +.. _Pereira2018.language: +Pereira2018.language +;;;;;;;;;;;;;;;;;;;; + + Citation: :cite:label:`pereira2018toward` + + + +.. _Pereira2018.auditory: +Pereira2018.auditory +;;;;;;;;;;;;;;;;;;;; + + Citation: :cite:label:`pereira2018toward` + + +Wikitext +++++++++ +Location: `data/wikitext +`_ + + +.. _wikitext-2/test: wikitext-2/test -+++++++++++++++ -brainscore_language/data/wikitext +;;;;;;;;;;;;;;; + Metrics ~~~~~~~ -pearsonr +The following Metrics are available. Plugin identifiers are grouped by +directory, shown in bold. + +Accuracy ++++++++ -brainscore_language/metrics/pearson_correlation +Location: `metrics/accuracy +`_ + + +.. _accuracy: +accuracy +;;;;;;;; + + +Linear_predictivity ++++++++++++++++++++ +Location: `metrics/linear_predictivity +`_ +.. _linear_pearsonr: linear_pearsonr -+++++++++++++++ -brainscore_language/metrics/linear_predictivity +;;;;;;;;;;;;;;; -accuracy -++++++++ -brainscore_language/metrics/accuracy +Pearson_correlation ++++++++++++++++++++ +Location: `metrics/pearson_correlation +`_ + + +.. _pearsonr: +pearsonr +;;;;;;;; + Models ~~~~~~ +The following Models are available. Plugin identifiers are grouped by +directory, shown in bold. + +Glove ++++++ +Location: `models/glove +`_ + + +.. _glove-840b: +glove-840b +;;;;;;;;;; + + +Gpt ++++ +Location: `models/gpt +`_ + + +.. _distilgpt2: distilgpt2 -++++++++++ -brainscore_language/models/gpt +;;;;;;;;;; + +.. _gpt2-xl: gpt2-xl -+++++++ -brainscore_language/models/gpt +;;;;;;; -glove-840b -++++++++++ -brainscore_language/models/glove + +.. _gpt-neo-2.7B: +gpt-neo-2.7B +;;;;;;;;;;;; + + + +.. _gpt-neo-1.3B: +gpt-neo-1.3B +;;;;;;;;;;;; + + +Earley_parser ++++++++++++++ +Location: `models/earley_parser +`_ + + +.. _earley-parser-minivocab: +earley-parser-minivocab +;;;;;;;;;;;;;;;;;;;;;;; + + +Lm1b +++++ +Location: `models/lm1b +`_ + + +.. _lm1b: +lm1b +;;;; + + +Random_embedding +++++++++++++++++ +Location: `models/random_embedding +`_ + + +.. _randomembedding-1600: +randomembedding-1600 +;;;;;;;;;;;;;;;;;;;; + + + +.. _randomembedding-100: +randomembedding-100 +;;;;;;;;;;;;;;;;;;; + + +Rnng +++++ +Location: `models/rnng +`_ + + +.. _rnn-slm-ptb: +rnn-slm-ptb +;;;;;;;;;;; + + + +.. _rnn-tdg-ptb: +rnn-tdg-ptb +;;;;;;;;;;; + + + +.. _rnn-slm-ptboanc: +rnn-slm-ptboanc +;;;;;;;;;;;;;;; + + + +.. _rnn-tdg-ptboanc: +rnn-tdg-ptboanc +;;;;;;;;;;;;;;; + + + +.. _rnn-slm-ptboanc-1024: +rnn-slm-ptboanc-1024 +;;;;;;;;;;;;;;;;;;;; + + + +.. _rnn-tdg-ptboanc-1024: +rnn-tdg-ptboanc-1024 +;;;;;;;;;;;;;;;;;;;; + + +Rnnlm ++++++ +Location: `models/rnnlm +`_ + + +.. _rnn-lm-ptb: +rnn-lm-ptb +;;;;;;;;;; Bibliography