diff --git a/.gitignore b/.gitignore index c18dd8d..f7a73aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,234 @@ +# Project-specific: logs and data are gitignored except .gitkeep (see AGENT.md) +/logs/* +!/logs/.gitkeep +corpora-in-progress +conf27/inductive-nlg-for-shared-tasks/corpus-creation/raw-data/ + + +# Byte-compiled / optimized / DLL files __pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +*.lcov +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +# Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +# poetry.lock +# poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +# pdm.lock +# pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +# pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi/* +!.pixi/config.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule* +celerybeat.pid + +# Redis +*.rdb +*.aof +*.pid + +# RabbitMQ +mnesia/ +rabbitmq/ +rabbitmq-data/ + +# ActiveMQ +activemq-data/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +# .idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ +# Temporary file for partial code execution +tempCodeRunnerFile.py + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# Streamlit +.streamlit/secrets.toml + +AGENT.md +PLAN.md +SKILL.md + +# Generated and fetched data stays outside Git. +/data/ diff --git a/conf27/inductive-nlg-for-shared-tasks/README.md b/conf27/inductive-nlg-for-shared-tasks/README.md new file mode 100644 index 0000000..7c20359 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/README.md @@ -0,0 +1,32 @@ +# Inductive Natural Language Generation for Shared Tasks + +## Overview + +This project investigates how AI agents can support shared tasks as human-driven experiments. The goal is not to automate shared tasks, but to study how agents can help researchers understand and communicate a solution space by synthesizing example solutions into an accessible synopsis. + +## Proposed Challenge + +The primary task is to generate a shared-task overview from participating systems' notebook papers, submissions, and—where available—prompt or communication logs. A complementary task may generate notebook papers directly from submissions, providing practical value while avoiding train–test leakage. + +Expected corpus structure: + +```text +task/ +├── papers/ +│ ├── paper-1.pdf +│ └── paper-n.pdf +├── overview.pdf +└── summaries.json +``` + +Systems will be evaluated through qualitative expert review and comparative automated judging. The evaluation must account for overview papers that describe teams without corresponding notebook papers. + +## Data Sources + +Candidate collections include shared tasks from [CLEF](https://clef-initiative.eu/), TREC, NTCIR, FIRE, SemEval, PAN, and Touché. Existing resources include the [TIRA shared-task collection](https://git.webis.de/code-research/tira/tira-shared-tasks) and canonical CLEF overview/notebook-paper proceedings. + +## Resources + +- [Proposal draft](https://www.overleaf.com/7375677649nhbknmygttkd#56811b) +- [UniAgent code and baselines](https://github.com/uniagent-webis-de/uniagent-code) +- [Archived corpus repository](https://github.com/uniagent-webis-de/uniagent_inlg_26-ARCHIVED) diff --git a/conf27/inductive-nlg-for-shared-tasks/baseline-naive/Dockerfile b/conf27/inductive-nlg-for-shared-tasks/baseline-naive/Dockerfile new file mode 100644 index 0000000..7a221a5 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/baseline-naive/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3-slim + +ADD requirements.txt / + +RUN pip3 install --no-cache-dir -r /requirements.txt \ + && pip3 --no-cache-dir install pytest \ + && rm /requirements.txt + +ADD *.py / + +RUN cd / \ + && PYTHONPATH=/ pytest test_baseline.py + diff --git a/conf27/inductive-nlg-for-shared-tasks/baseline-naive/README.md b/conf27/inductive-nlg-for-shared-tasks/baseline-naive/README.md new file mode 100644 index 0000000..3ba2839 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/baseline-naive/README.md @@ -0,0 +1,17 @@ +# Naive Baseline + +This deterministic baseline extracts each `paper.txt.md` title and abstract. + +```bash +python3 baseline.py \ + --input ../corpora-in-progress/touche-20-task-1-spot-check/papers \ + --output predictions.jsonl \ + --summary title-and-abstract +``` + +`--summary` accepts `title`, `abstract`, or `title-and-abstract`. The output +contains one JSON object per line: + +```json +{"id": "172", "summary": "Argument Retrieval Using Deep Neural Ranking Models"} +``` diff --git a/conf27/inductive-nlg-for-shared-tasks/baseline-naive/baseline.py b/conf27/inductive-nlg-for-shared-tasks/baseline-naive/baseline.py new file mode 100755 index 0000000..f5648d5 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/baseline-naive/baseline.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 + +import json +import re +from enum import Enum +from pathlib import Path + +import click + + +class SummaryMode(str, Enum): + TITLE = "title" + ABSTRACT = "abstract" + TITLE_AND_ABSTRACT = "title-and-abstract" + + +def normalize_text(text: str) -> str: + text = re.sub(r"[*_`]+", "", text) + text = re.sub(r"(?<=\w)-\s+(?=[a-z])", "", text) + return re.sub(r"\s+", " ", text).strip() + + +def extract_title(markdown: str) -> str: + match = re.search(r"^#\s+(.+?)\s*$", markdown, flags=re.MULTILINE) + if not match: + raise ValueError("No level-one title found") + return normalize_text(match.group(1)) + + +def extract_abstract(markdown: str) -> str: + lines = markdown.splitlines() + abstract_lines: list[str] = [] + in_abstract = False + + for line in lines: + if not in_abstract: + match = re.match( + r"^\s*(?:#{1,6}\s*)?(?:\*{1,2})?Abstract" + r"(?:\*{1,2})?\s*(?:[.:]\s*)?(.*)$", + line, + flags=re.IGNORECASE, + ) + if match: + in_abstract = True + abstract_lines.append(match.group(1)) + continue + + if re.match(r"^\s*#{1,6}\s+", line): + break + if re.match(r"^\s*(?:\*{0,2})?(?:Keywords|Copyright)\b", line): + break + abstract_lines.append(line) + + abstract = normalize_text("\n".join(abstract_lines)) + if not abstract: + raise ValueError("No abstract found") + return abstract + + +def create_summary(title: str, abstract: str, mode: SummaryMode) -> str: + if mode is SummaryMode.TITLE: + return title + if mode is SummaryMode.ABSTRACT: + return abstract + return f"{title}\n\n{abstract}" + + +def find_papers(input_directory: Path) -> list[Path]: + papers = sorted(input_directory.rglob("paper.txt.md")) + if not papers: + raise ValueError(f"No paper.txt.md files found in {input_directory}") + + ids = [paper.parent.name for paper in papers] + if len(ids) != len(set(ids)): + raise ValueError("Paper directory names must be unique") + return papers + + +def generate_summaries( + input_directory: Path, output_file: Path, mode: SummaryMode +) -> None: + papers = find_papers(input_directory) + output_file.parent.mkdir(parents=True, exist_ok=True) + + with output_file.open("w", encoding="utf-8") as output: + for paper in papers: + markdown = paper.read_text(encoding="utf-8") + title = extract_title(markdown) + abstract = extract_abstract(markdown) + record = { + "id": paper.parent.name, + "summary": create_summary(title, abstract, mode), + } + output.write(json.dumps(record, ensure_ascii=False) + "\n") + + +@click.command() +@click.option( + "--input", + "input_directory", + required=True, + type=click.Path(exists=True, file_okay=False, path_type=Path), + help="Directory containing paper directories with paper.txt.md files.", +) +@click.option( + "--output", + "output_file", + required=True, + type=click.Path(dir_okay=False, path_type=Path), + help="Destination JSONL file.", +) +@click.option( + "--summary", + "summary_mode", + required=True, + type=click.Choice([mode.value for mode in SummaryMode]), + help="Text used as each paper's summary.", +) +def main(input_directory: Path, output_file: Path, summary_mode: str) -> None: + try: + generate_summaries( + input_directory, output_file, SummaryMode(summary_mode) + ) + except ValueError as error: + raise click.ClickException(str(error)) from error + + +if __name__ == "__main__": + main() diff --git a/conf27/inductive-nlg-for-shared-tasks/baseline-naive/requirements.txt b/conf27/inductive-nlg-for-shared-tasks/baseline-naive/requirements.txt new file mode 100644 index 0000000..dca9a90 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/baseline-naive/requirements.txt @@ -0,0 +1 @@ +click diff --git a/conf27/inductive-nlg-for-shared-tasks/baseline-naive/test_baseline.py b/conf27/inductive-nlg-for-shared-tasks/baseline-naive/test_baseline.py new file mode 100644 index 0000000..e035bcf --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/baseline-naive/test_baseline.py @@ -0,0 +1,86 @@ +import json + +from click.testing import CliRunner + +from baseline import ( + SummaryMode, + create_summary, + extract_abstract, + extract_title, + main, +) + + +def test_extracts_title_and_heading_abstract(): + markdown = """# Paper Title + +Author + +### Abstract This is a hyphen- +ated abstract. + +## 1 Introduction + +Not part of the abstract. +""" + + assert extract_title(markdown) == "Paper Title" + assert extract_abstract(markdown) == "This is a hyphenated abstract." + + +def test_extracts_plain_and_bold_abstracts(): + plain = """# First + +Abstract. First paragraph. + +Second paragraph. + +Keywords: example +""" + bold = """# Second + +**Abstract A bold abstract** + +## Introduction +""" + + assert extract_abstract(plain) == "First paragraph. Second paragraph." + assert extract_abstract(bold) == "A bold abstract" + + +def test_summary_modes(): + assert create_summary("Title", "Abstract", SummaryMode.TITLE) == "Title" + assert create_summary("Title", "Abstract", SummaryMode.ABSTRACT) == "Abstract" + assert ( + create_summary("Title", "Abstract", SummaryMode.TITLE_AND_ABSTRACT) + == "Title\n\nAbstract" + ) + + +def test_cli_writes_jsonl(tmp_path): + input_directory = tmp_path / "papers" + paper_directory = input_directory / "172" + paper_directory.mkdir(parents=True) + (paper_directory / "paper.txt.md").write_text( + "# Paper Title\n\n### Abstract Paper abstract.\n\n## Introduction\n", + encoding="utf-8", + ) + output_file = tmp_path / "output" / "summaries.jsonl" + + result = CliRunner().invoke( + main, + [ + "--input", + str(input_directory), + "--output", + str(output_file), + "--summary", + "abstract", + ], + ) + + assert result.exit_code == 0 + assert json.loads(output_file.read_text(encoding="utf-8")) == { + "id": "172", + "summary": "Paper abstract.", + } diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/README.md b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/README.md new file mode 100644 index 0000000..1af711a --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/README.md @@ -0,0 +1,44 @@ +--- +configs: +- config_name: inputs + data_files: + - split: train + path: + - "papers/**/**" +- config_name: truths + data_files: + - split: train + path: + - "manual-paper-summaries.jsonl" + +tira_configs: + resolve_inputs_to: "." + resolve_truths_to: "." + default_upload_name: "predictions.jsonl" + input_format: + name: "arbitrary" + truth_format: + name: "*.jsonl" + config: + id_field: "id" + value_field: "summary" + required_fields: ["id", "summary"] + minimum_lines: 5 + baseline: + link: "../../baseline-naive/" + command: "/baseline.py --input $inputDataset --summary title-and-abstract --output ${outputDir}/predictions.jsonl" + format: + name: "*.jsonl" + evaluator: + image: "mam10eks/uniagent-inlg:evaluator-0.0.1" + command: "/evaluate.py --predictions ${inputRun}/predictions.jsonl --truths ${inputDataset}/manual-paper-summaries.jsonl --results ${outputDir}" +--- + + +```bash +tira-cli dataset-submission \ + --path touche-20-task-1-spot-check \ + --task uniagent-2026 \ + --split train \ + --dry-run +``` diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/manual-paper-summaries.jsonl b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/manual-paper-summaries.jsonl new file mode 100644 index 0000000..d1727da --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/manual-paper-summaries.jsonl @@ -0,0 +1,6 @@ +{"id": "172", "title": "Argument Retrieval Using Deep Neural Ranking Models", "authors": "Saeed Entezari and Michael Völske", "summary": "Aragorn by Entezari and Völske employs a distant-supervision approach to train seven deep-learning models, the idea being that the argument retrieval task is similar to the structure of arguments in the args.me corpus: Retrieving relevant arguments for a query is comparable to retrieving premises that support an argument's conclusion. Models receive the conclusion as a query during the training process and rank possible premises, with premise/conclusion labels derived from the corpus. To construct training samples, the premise text of each argument is marked as relevant, whereas the top 100 premises from other arguments ranked by BM25 are marked as irrelevant. Using that methodology, a distant-supervision dataset is produced comprising over 300,000 training and 4,800 validation queries to train the seven deep neural models. The training comprises multiple epochs per model, using only the best-performing candidate per model for the actual argument retrieval task, according to MAP@20 on the validation queries. All seven trained models are combined using linear regression optimized on the validation queries. A run was submitted for this combined approach (Run 3), with additional individual runs for the four most promising trained models (Runs 1, 2, 4, and 5). For all five runs, the retrieval pipeline used BM25 to retrieve 100 arguments and then re-ranked them by the predicted model score."} +{"id": "174", "title": "Ranking Arguments by Combining Claim Similarity and Argument Quality Dimensions", "authors": "Lorik Dumani and Ralf Schenkel", "summary": "Don Quixote by Dumani and Schenkel follows a two-step approach: In an initial offline operation, the dataset is clustered based on Sentence-BERT (SBERT) embeddings of both the conclusions and the premises. Premises are further grouped by identical conclusions. This grouping allows to calculate the so-called dimension convincing frequencies (DCFs) of a premise, in comparison to other premises of the same conclusion. For the three quality dimensions cogency, reasonableness, and effectiveness, a logistic regression is applied to all possible conclusion-premise pairs. The DCFs are then obtained by counting how often a premise was better than other premises belonging to the same conclusion in a cross comparison. Conclusions and premises are indexed separately. At retrieval time, a set of premises is retrieved using a divergence from-randomness model. This set is then extended with all other premises belonging to the same group. For each entry in the extended set, a score is calculated using both the similarity of the query to the conclusion and the sum of the three DCFs per premise. For each group in this result set, the scores of its members are aggregated and a final ranking is obtained by choosing a single representative per group, and ranking representatives by group score. Representatives are chosen by text length, under the hypothesis that a longer premise is also more specific and therefore may be better suited."} +{"id": "241", "title": "Exploring Argument Retrieval with Transformers", "authors": "Christopher Akiki and Martin Potthast", "summary": "Dread Pirate Roberts by Akiki and Potthast employs transformer-based models as part of its argument retrieval pipeline, pursuing three independent approaches: (1) The initial query is expanded using GPT-2; by adding question-like suffixes to the query, the text generation is steered towards argumentative text. Thus, a set of queries is built from generated sentences, and for each, results are retrieved using a DirichletLM retrieval model. Finally, all results are combined and ranked by their respective absolute score (Run 1). (2) Similar to the first approach, query expansion is achieved by generating argumentative text. However, instead of generating sentences, single word predictions by BERT are used. Once again, question-like suffixes are employed to influence the nature of the generated text. Final results are obtained by composing a query out of the single generated terms and retrieving results with a DirichletLM model (Run 2). (3) Instead of focusing on query expansion, the third approach uses a transformer-based model to obtain document representations. Arguments are embedded in a vector space using Google's BERT-like Universal Sentence Encoder (USE). Retrieval is subsequently conducted using nearest-neighbor search with respect to the query (Runs 3, 4, and 5)."} +{"id": "191", "title": "SentArg: A Hybrid Doc2Vec/DPH Model with Sentiment Analysis Refinement", "authors": "Christian Staudte and Lucas Lange", "summary": "Oscar François de Jarjayes by Staudte and Lange combines the traditional DPH retrieval model with document similarities based on a CBOW dual embedding space model. The intersection of the top-1000 result sets of both retrieval strategies is ranked descending by DPH score, producing a search result that ensures contextual relevance, as determined by the dual embedding space model, as well as query-specific accuracy, as given by the DPH scoring. Furthermore, a sentiment-based score weighting is proposed under the hypothesis that texts written by emotionally involved authors (high sentiment values according to the Google Cloud Natural Language API) are of higher quality and relevance to a query than neutral documents. An evaluation based on the retrieval performance of different sentiment weighting schemes seems to support this hypothesis."} +{"id": "182", "title": "Creating an Argument Search Engine for Online Debates", "authors": "Maximilian Bundesmann, Lukas Christ, and Matthias Richter", "summary": "Weiss Schnee by Bundesmann et al. integrate a notion of argument quality as well as result heterogeneity into the argument retrieval process. First, quality ratings for all arguments in the args.me corpus are predicted using Support Vector Regression (SVR) based on 22 text features. For retrieval, the authors considered three different strategies of query expansion, one based on WordNet synonyms, the other two using embedding-based language modeling. Based on the augmented queries, an initial set of arguments is retrieved and scored by a DPH retrieval model, additionally weighted by their quality score. The top-8 results are then re-ranked, maximizing the weighted sum of semantic distance between consecutive entries in the result list, thus ensuring result heterogeneity, as well as each entries' argumentative quality. However, this reranking approach is considered an experimental feature and is not included in officially submitted runs, as the nDCG evaluation measure does not take diversity of results into account."} +{"id": "207", "title": "University of Amsterdam at CLEF 2020", "authors": "Mahsa S. Shahshahani and Jaap Kamps", "summary": "Zorro by Shahshahani and Kamps was evaluated with only a single run. First, document encodings are constructed using BERT on the first 512 tokens of each argument in the args.me corpus. A ranking is then created in three steps. (1) An initial ranking is produced using BM25. (2) An SVM classifier trained on the Dagstuhl-15512 ArgQuality corpus is used to predict scores for cogency, well-writtenness, reasonableness and overall quality of each argument in the initial ranking. (3) The final ranking is then produced using the learning-to-rank library RankLib. In addition to the scores predicted in the previous step, training incorporates the following features: the BERT encoding, two named entity-based features, and two binary features indicating the presence of numerical named entities (percent, quantity, money) and other entities (person, location, organization) in the argument. The assumption is that an argument exhibiting such entities is more likely to provide users with persuasive and effective information."} diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_1.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_1.png new file mode 100644 index 0000000..b6e4a16 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_1.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_2.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_2.png new file mode 100644 index 0000000..ec0e332 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_2.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_3.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_3.png new file mode 100644 index 0000000..783d39f Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_3.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_4.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_4.png new file mode 100644 index 0000000..3839096 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_4.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_5.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_5.png new file mode 100644 index 0000000..8d4997b Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_5.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_6.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_6.png new file mode 100644 index 0000000..da0668d Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p10_6.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p8_1.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p8_1.png new file mode 100644 index 0000000..e75bda1 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p8_1.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p9_1.jpg b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p9_1.jpg new file mode 100644 index 0000000..6a99062 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/figures/img_p9_1.jpg differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/paper.pdf b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/paper.pdf new file mode 100644 index 0000000..1886262 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/paper.pdf differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/paper.txt.md b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/paper.txt.md new file mode 100644 index 0000000..98a6b88 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/paper.txt.md @@ -0,0 +1,240 @@ +# Argument Retrieval Using Deep Neural Ranking Models + +Saeed Entezari and Michael Völske + +Bauhaus Universität Weimar saeed.entezari@uni-weimar.de, michael.voelske@uni-weimar.de + +### Abstract Conversational argument retrieval is the problem of ranking argumen- + +tative texts in a collection of focused arguments in order of their relevance to a textual query on different topics. In this notebook-paper for Touché by taking a distant supervision approach for constructing the query relevance information, we investigate seven different deep neural ranking models proposed in the literature with respect to their suitability to this task. In order to incorporate the insights from multiple models into an argument ranking, we further investigate a simple linear aggregation strategy. By retrieving relevant arguments using deep neural ranking models, it will be inspected to what extent the systems whose main concentration is on relevant documents, would be able to retrieve arguments which meet various quality dimensions of the arguments. Our test results suggest that the interaction-focused networks provide better performance compared to the representation-focused networks. + +## 1 Introduction + +Arguments may have existed since humans first started communicating [4]. People use arguments in order to prove or contradict an opinion, in particular on controversial topics where opinions diverge widely. Rieke et al. define an argument as a unit composed of a claim (conclusion) and its supporting premises [10]. Generally, premises can support or attack a claim: the premises of one claim can be used to support or attack other claims. A conclusion could be a word, phrase or even a sentence. Typically the premises are texts composed of multiple sentences or paragraphs. Due to the variety of opinion towards controversial topics, a corresponding query typically does not have a single correct answer, and getting an exhaustive overview can take considerable time [14]. In this situation, a ranking model which can neutrally retrieve the arguments on all sides of a controversial topic can provide users with a reasonable approach toward difficult questions. Such argument retrieval systems can benefit debate support and writing assistance systems, as well as automated decision making and opinion summarization. This paper describes our contribution to the Touché 2020 shared task on conversational argument retrieval. By taking a distant supervision approach, our primary focus of investigation is on a variety of neural ranking models that have been proposed in the literature in recent years [3,8,15,16], and how they can apply to the conversational + +Copyright © 2020 for this paper by its authors. Use permitted under Creative Commons License Attribution 4.0 International (CC BY 4.0). CLEF 2020, 22-25 September 2020, Thessaloniki, Greece. + +----- + +argument retrieval setting. In our experiments, a basis retrieval model such as BM25 produces an initial ranking which is then re-ranked by the deep neural model (except in the case of end-to-end models, which operate without an initial retrieval). We compare seven different neural ranking models overall. In addition to tackling this problem with individual neural rankers, we also explore a simple rank aggregation scheme based on a linear combination of the models' scores. Based on the test results, interaction-focused networks outperform significantly the representation-focused networks. Using the contextualized embedding representation, the convergence in the training phase happens faster and a certain level of performance could be achieved. + +In what follows, we first review a selection of relevant related works on argumentation and argument retrieval, as well as the shared task setting. In Section 3, we briefly introduce the ranking models that comprise our study; we include recurrent siamese networks, kernel-based neural ranking models, different variants of contextualized embedding-based models, as well as stand-alone neural rankers. Section 4 explains our experimental setting including data preprocessing, training the various ranking models, as well as our aggregation setup, and Section 5 showcases our results. We conclude with a summary and discussion of our results in Section 6. . . . + +## 2 Background and Related Work + +Args.me, one of the first prototypes of an argument search engine [14] ranks arguments crawled from debate websites using the classical BM25F retrieval model. AR- + +GUMENTEX retrieves topic-related arguments from a large collection of web documents [11] in a three-stage approach: (1) retrieving relevant documents using BM25, (2) identifying arguments in those documents, and (3) classifying the arguments into pro and con. To evaluate an argument's convincingness, Habernal and Gurevych proposed the use neural networks [6]: using on annotator judgments on how convincing the arguments are, a bidirectional LSTM is trained to predict which of a given pair of arguments is more convincing. + +Dumani proposed a two-stage system for argument retrieval [4], which first retrieves the conclusions related to a given query, and then returns the premises associated with those conclusions. He suggested different similarity measures to semantically match conclusions to the query, such as plain language models with additional smoothing, and taking the textual context of the claim into account; these would be used to search through clusters of premises in the second stage. + +The criteria for ranking arguments can be categorized into three main groups related to different argument quality aspects [13]: Logical aspects focus on the soundness of the arguments; logical arguments will have acceptable premises relevant to their conclusions. Rhetorical aspects pertain to the ability to persuade [7], and evaluate how successful an argument is in persuading its target audience [1]. Dialectical aspects assess the degree to which an argument helps its recipients formulate their own stance on the topic-this may also be considered as the utility of the argument [13]. Our study focuses especially on retrieving arguments relevant to a given query, and as such we are mainly concerned with retrieving logical arguments. + +----- + +### 2.1 Touché task and Dataset + +The Touché @ CLEF shared task on Conversational Argument Retrieval (Task 1) targets a retrieval scenario in a focused argument collection to support argumentative conversations [2]. The focused argument collection in this case is the args.me corpus,1 which forms the setting for our study in combination with a collection of argumentative queries. While the arguments in this dataset are annotated with a stance, our models do not consider this for the purpose of evaluating their relevance to the given queries. + +## 3 Models + +Four categories of deep neural ranking models have been used in this study. Each category may include one or multiple network variations. For the all networks the hinge loss function (a pairwise loss function) which is typical for ranking tasks is used to train the models. Optimizing this loss function will contribute the models to put related documents over the unrelated ones. Note that except SNRM which is trained using TensorFlow 1.3, the rest of networks have been trained and validated in PyTorch 1.2. The models were trained on 7 different GPUs in parallel and took a day to get all models trained. The inference phase of all models can reproduced in the TIRA platform [9] and takes half an hour and 4 to 5 hours for the case of classical and contextualized embedding respectively. Due to the lack of GPU, reproducing the training results in TIRA would take a long time. + +### 3.1 Recurrent Based Siamese Model + +For the purpose of investigating the representation-based networks in the task of argument retrieval we have used Siamese network which are typically used for producing similarity score. Gated Recurrent Units (GRU) are used to produce representation of query and documents. The concatenation of the query and the document representations are then fed to a linear layer to produce a similarity score [12]. Bidirectional units with a hidden size of 512 have been used for GRU units and the linear layer is a fully connected network with an input size of 4 × 512 to 1 (the concatenation of two bidirectional units produces an output with the dimensionality of 4 times of the hidden state). + +### 3.2 Kernel Based Neural Ranking Models + +The Kernel based Neural Ranking Models (KNRM) aims to produce a similarity score for a given query and document pair by focusing on modeling the interaction that they have using RBF kernels. This model is composed of three important parts: translation model, kernel pooling, and learning to rank model [15]. The similarity score is produced by a fully connected learning-to-rank layer. The input of this layer is the result of applying RBF kernels to each row of a translation matrix whose elements are the cosine + +1 https://webis.de/data/args-me.html + +----- + +similarities of the query and the document terms. The original implementation of the kernel based models and Siamese network is available 2 . + +Another variation of the kernel based neural ranking model used in this study is convolutional KNRM (Conv-KNRM). The most important difference between this network and KNRM is the use of a set of convolutional filters to form different n-gram embeddings. In the cross-matching layer, the similarity of the query and the document n-grams is calculated using cosine similarity [3]. Kernel pooling, the learning-to-rank layer, and the cost function are the same as for the previous network. + +### 3.3 Contextualized Embedding for Ranking + +The Contextualized Embeddings for Document Ranking (CEDR) model aims to improve ranking performance with the help of a deeper understanding of text semantics [8]. Unlike the traditional word embeddings such as word2vec or GloVe, contextualized language models consider the contexts of each word occurrence in order to assign it an embedding. For instance, the word bank may have different representations in different sentences depending on the context it occurs in. + +Among the contextualized embedding techniques, BERT has proven to be one of the best performing in different NLP tasks. Through its ability to encode multiple text segments, BERT allows us to make informed judgments about the similarity of text pairs [8]. In this study we have used the BERT-base uncased model which produces a vector of 768 dimensions for the tokens. The original implementation of the networks which have used contextualized embedding can be found in GitHub 3 . + +### Vanilla BERT Compared to the other deep neural ranking models using contextual- + +ized embedding, a relatively simple ranking model is obtained by the fine-tuning of the BERT model with a linear layer stacked at top [8]. During training, this linear layer requires a relatively larger learning rate than the pretrained BERT weights, which we only want to adjust slightly. + +### BERT and DRMM The language model knowledge encoded in the contextualized + +embeddings can be combined with any existing neural ranking model simply by stacking it on top of the BERT model [8]. One of the deep ranking models that we have used in this role is the DRMM model to see how the performance will change [5]. As the DRMM on its own did not represent a convincing performance on the validation set, we have excluded its result from reporting. + +### BERT and KNRM As an alternative to DRMM, we also combine the aforementioned + +KNRM model with the contextualized embedding. In our study, we use KNRM with static embedding, i.e. the BERT weights are not adjusted at all during training in this case. As we have already trained KNRM with static embedding, this setting will give us a good illustration of how the pretrained contextualized embedding will effect the performance of the model. + +2 https://github.com/thunlp/Kernel-Based-Neural-Ranking-Models/tree/master/src 3 https://github.com/Georgetown-IR-Lab/cedr + +----- + +### 3.4 Stand Alone Neural Ranking Models + +All the networks that have been discussed up to now require a small set of candidate documents for re-ranking, which must be provided by a traditional retrieval model. As such, the performance of the model is limited by what the first-stage ranker (in our case BM25) can provide. By contrast, the stand-alone neural ranking model (SNRM) builds an inverted index from a latent sparse representation of the input document collection, which is searched directly with a corresponding representation of the query. This representation is achieved by an hour-glass shaped fully-connected network, and captures the semantic relationships between the query and documents [16]. During retrieval, SNRM finds those documents whose representations have non-zero in the same positions as the query; hence, the sparser the query representation, the faster the retrieval will be [16]. For this reason, the SNRM training procedure optimizes a traditional hinge loss term in combination with a sparsity objective. The original implementation of the network in TensorFlow can be found in GitHub 4 . + +## 4 Experiments + +This section discusses the experiments of this study. In order to do an ad-hoc retrieval task we require the relevance information of the query and document pairs known as qrel file which can be derived from the click-through or query log information. In the provided dataset in Touché task however, we have just the annotation of the argument components. Thanks to the distant supervision that we have taken, we consider the annotated premise of each argument as a related document to the conclusion of the argument, which is considered as a query in the collection. For a typical ranking task, we still require unrelated documents to the queries. By using fuzzy similarity between the queries (conclusions), we assign the corresponding premise of the unrelated conclusions (conclusions with less fuzzy similarity score) to each argument. This way we form a binary version of qrel information for the dataset and prepare it to train ranking models for the task of ad-hoc retrieval argument task on it. + +### 4.1 Training and Validation Data + +We believe that the arguments whose premise lengths are less than 15 tokens could not be considered as convincing and good arguments. As a result we set aside such arguments. We have split the dataset into training and validations set. After the preprocessing step we are left with 312248 training and 4885 validation arguments. We tried to keep the validation set small in order to incorporate more information in the training phase while still allowing a meaningful assessment of model performance during validation. Note that we have selected the arguments with exactly 5 premises to be in validation set. According to the distant super vision approach, these premises would be the related documents to the conclusion of the argument. For each argument we assigned 100 unrelated premises. + +4 https://github.com/hamed-zamani/snrm + +----- + +As the preprocessing phase of the contextualized embedding networks is a bit different (in contrast to the static embedding, in contextualized embedding the punctuation do not require to be tokenized) we formed two separate training and validation set for these networks. Note that the training and validation arguments are the same for these sets so that the results could be comparable. + +### 4.2 Model Training + +We keep the batch size to 32 for different networks. For all the networks, in order to have 8 evaluations per epoch, after 1239 training batches we run the validation to evaluate the performance of the network and if the MAP@20 measure was better than the best result obtained so far, the saved model is replaced correspondingly. As the query relevance information that we have formed for the dataset is in a binary format, we believe that MAP@20 would be a better evaluation measure compared to nDCG@20 as it is designed mostly for the soft similarity score of relevance. We run the different networks for 10 epochs. For the models with contextualized embedding, as the curves suggest, there is no need to train for this many epochs. We have trained them for 5 epochs. This saves the time and avoids complex computations out of which we do not get noticeable improvement. The average error and validation curves for different networks are displayed for every evaluation that we have done on the validation set. Note that the validation points are displayed in percentage and the coordinates of the best MAP@20 achieved in the corresponding run (the step number and the MAP@20 value) have been written displayed on the MAP curve with a blue dot. + +### Recurrent Network We keep the dimensionality for the input tokens to 100 and the + +learning rate to 0.001. The hidden size for the GRUs have been selected to be 512. For the linear layer we have the dropout layer with the rate of 0.5. + +### KNRM We decided to have 21 bins for this network as it was suggested by Xiong et + +al. [15]. Learning rate and word embedding dimensionality are as the same as recurrent network. + +### CKNRM The parameters for the network are the same as for KNRM model. Con- + +volutional layers are 2D filters whose input is of dimension 1 and the output has the dimensionality of 128. The window sizes of the convolution layers are 1, 2, and 3 as suggested by Dai et al. [3]. The ReLu activation function has been applied on the output of the convolutional layers. + +### Vanilla BERT The learning rate for the BERT layers are much smaller than for the + +linear layer as we do not intend to make large changes to the pretrained contextualized embedding. We keep the learning rate of the BERT layers to be 2 ∗ 10-5 and for the linear layer the learning rate is 10-3 . For the purpose of generalization we add a dropout layer with the probability of 0.1. The linear layer has the input size of 768 to 1. 768 is the embedding dimensionality for a token in BERT model. + +----- + +Table 1: Best achieved evaluation scores of the models + +Metrics @20 + +| Model | MRR MAP nDCG | +|---|---| +| GRU | 28.4 24.1 38.05 | +| KNRM | 84.35 72.64 80.24 | + +Conv-KNRM 86.72 73.32 82.08 SNRM 82.41 70.14 78.97 Vanilla BERT 95.12 88.5 91.00 KNRM BERT 94.57 90.18 89.80 DRMM BERT 95.97 88.09 91.34 + +### BERT and DRMM The learning rates for the BERT and non-BERT layers are the + +same as the Vanilla BERT. The number of bins is 11. For the feed-forward network we exploited 2 hidden layers of 256 and 5 units. + +### BERT and KNRM The Learning rate for the fine tuning of the BERT layers and + +training the KNRM layers are kept the same as for the Vanilla BERT model. The number of bins is 11 and the parameters for RBF functions are kept as what was suggested by the authors as the results on the sample data were acceptable. + +### SNRM For this model we did not use any hidden layer and it showed reasonable de- + +crease of cost function on the training set. Learning rate is selected to be 10-4 and no drop out was used. We have trained all the models in parallel on 8 GPUs. Table 1 shows the best evaluation scores achieved by different models. + +### Aggregation Now that we have the retrieved documents from each model, we can ag- + +gregate the results by producing a score which is the result of linear aggregation of the model scores. As the first step of aggregation, we analyze how diverse the result of the networks are. This would give a hint how reliable the network results are. Figure 1 illustrates two measures of ranking diversity namely Jaccard and Spearman. Considering the network results for the retrieved documents as vectors with the dimensionality of the retrieved documents and values of ranking score, we took the mean of the Jaccard and Searman measures over the 50 test queries for illustrating how diverse the result of the networks from each other are. We decided to exclude SNRM in the aggregation as its results are diverse from the rest of the models. + +The linear regression is trained on the model results for the validation set. The trained model is then applied on the document scores for the test queries achieved from different models. All the model scores have been normalized to be in the same range. + +----- + +![](../figures/paper_172/img_p8_1.png) + +Figure 1: The heat map of the Jaccard (upper) and Spearman (lower) correlation coefficient for the 50 test queries + +### 4.3 Test Queries + +After training the models and getting the best one from the validation phase, it is time to give the models the test queries and see what documents would be ranked top. Except the SNRM model which has generated inverted index and can retrieve the documents on its own, other networks require to be provided with candidate documents (premises). To this end we make use of BM25. + +We first group all the arguments based on the normalized conclusion column. Using BM25 we retrieve the most relevant normalized conclusions. We select the top 100 normalized conclusions. The premises corresponding to retrieved normalized conclusions are the candidate documents to be ranked by the neural networks. Note that each of the normalized conclusion may have a different number of premises. Consequently, the number of documents to be ranked may vary for different test queries. Figure 2 shows how we provide the trained networks with the document-query pairs to rank in the test phase. After getting the document scores, we sort them based on the score in a descending way. We introduce the top 100 premises as the retrieved arguments for each test query. There are 50 test queries which results in 5000 retrieved arguments by each model. + +----- + +![](../figures/paper_172/img_p9_1.jpg) + +Figure 2: Candidate documents to be re-ranked in the test phase + +## 5 Results + +**Training and validation results Figure 3 showcases the training progress for a selec-** + +tion of the models in our study. Each subplot comprises a learning curve in the top half, which shows the development of the training loss over the epochs shown along the xaxis. Note that the mean of the error over 1239 batches are represented. The bottom half of each plot shows the development of the retrieval performance on the validation set performed for every 1239 training batch-as measured in terms of mean reciprocal rank (MRR), mean average precision (MAP), and normalized discounted cumulative gain (nDCG)-over the same time steps. The plots highlight that the contextual-embedding based retrieval models (Figures 3d, 3e, and 3f) converge faster, and achieve better validation performance than the other models (Figure 3a, 3b, and 3c) that don't incorporate contextual-embedding information. + +**Test Results Table 2 shows the performance of the different models in the test phase** + +provided by Touché committee. We assume that the models whose test results are not provided did not achieve better score than the displayed scores. The nDCG@5 has been reported as the test results. Evaluation of the retrieved arguments is done by human annotators based on the argument quality dimensions discussed by Wachsmus et.al in [13]. Devising the strategies for mapping the interaction of the input pairs may result in more promising models in the ad-hoc tasks. Represent-focused networks cannot have a good performance in retrieving relative arguments as they overlook the interaction of the input pairs. KNRM achieved the best score and ranked fourth among the competitors of the shared task. Exploiting the contextualized embedding contributes to achieve a certain level of test score which can be improved by devising more intuitive structures on the BERT weights. The best models with the best validation scores are not the best ones in the test phase. This may due to some facts: in the validation we focused on the top 20 retrieved documents while in the test phase, top 5 hits are targeted for each model. Furthermore, in the validation phase the models had to rank 105 premises. For the re-ranking in the test phase, however, this number is much larger ranging from 150 to 1200 arguments. Consequently, it is not surprising that the test scores would be of + +----- + +![](../figures/paper_172/img_p10_1.png) + +![](../figures/paper_172/img_p10_2.png) + +(a) Recurrent Based Siamese (b) KNRM + +![](../figures/paper_172/img_p10_3.png) + +![](../figures/paper_172/img_p10_4.png) + +(c) CKNRM (d) Vanilla BERT + +![](../figures/paper_172/img_p10_5.png) + +![](../figures/paper_172/img_p10_6.png) + +(e) BERT and DRMM (f) BERT and KNRM + +Figure 3: Training and Validation curves + +----- + +a lower grade. It can be interpreted that not necessarily related arguments would meet the other argument quality dimensions. Comparing the results of the validation and the test phase, highlights the importance of acquiring a dataset by which, developing the models for retrieving arguments meeting the other argument quality dimensions is possible. Considering the fact that only a few models of the competitors outperform the baseline method (Dirichlet LM with the score of 75.6%) reflects that retrieving the arguments meeting the quality dimensions of the arguments is not a trivial task. + +Table 2: Test scores of the models + +Model nDCG@5 (%) + +| GRU | x | +|---|---| +| DRMM | x | +| KNRM | 68.4 | +| CKNRM | x | +| SNRM | x | + +Vanilla BERT 40.4 KNRM BERT 31.9 DRMM BERT 37.1 Aggregation 37.2 + +## 6 Discussion + +In this study, thanks to taking a distant supervision technique, we used the deep neural ranking models to retrieve the most relevant arguments to the given queries provided in the Touché shared task. Test results suggest that focusing on the interaction of the inputpairs would contribute to more promising results in the ad-hoc retrieval task. KNRM achieved the best test results and ranked fourth among the competitors. Exploiting the contextualized embedding will result in achieving a certain level of score, a more intuitive structure is still required for better results. A mathematical expression of the argument quality dimensions to be included in the cost function of the models seems to be a primary step that should be taken for the task of argument retrieval. As the relevant arguments are not necessarily the ones which meet the other argument quality measures, developing a dataset including the information regarding to the different argument quality dimensions along side the relevance information is mandatory for developing the models with good retrieved arguments. A long way for devising an end-to-end neural ranking model for retrieving acceptable arguments exists to get a reliable results for the task of argument retrieval. + +----- + +## References + +1. Blair, J.A.: Groundwork in the theory of argumentation: Selected papers of J. Anthony Blair, vol. 21. Springer Science & Business Media (2011) +2. Bondarenko, A., Fröbe, M., Beloucif, M., Gienapp, L., Ajjour, Y., Panchenko, A., Biemann, C., Stein, B., Wachsmuth, H., Potthast, M., Hagen, M.: Overview of Touché 2020: Argument Retrieval. In: Working Notes Papers of the CLEF 2020 Evaluation Labs (Sep 2020) +3. Dai, Z., Xiong, C., Callan, J., Liu, Z.: Convolutional neural networks for soft-matching ngrams in ad-hoc search. In: Proceedings of the eleventh ACM international conference on web search and data mining. pp. 126-134 (2018) +4. Dumani, L.: Good premises retrieval via a two-stage argument retrieval model. In: Grundlagen von Datenbanken. pp. 3-8 (2019) +5. Guo, J., Fan, Y., Ai, Q., Croft, W.B.: A deep relevance matching model for ad-hoc retrieval. In: Proceedings of the 25th ACM International on Conference on Information and Knowledge Management. pp. 55-64 (2016) +6. Habernal, I., Gurevych, I.: Which argument is more convincing? analyzing and predicting convincingness of web arguments using bidirectional lstm. In: Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers). pp. 1589-1599 (2016) +7. Kennedy, G.A.: Aristotle, on Rhetoric: A Theory of Civic Discourse, Translated with Introduction, Notes and Appendices. Oxford: Oxford University Press (2007) +8. MacAvaney, S., Yates, A., Cohan, A., Goharian, N.: Cedr: Contextualized embeddings for document ranking. In: Proceedings of the 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval. pp. 1101-1104 (2019) +9. Potthast, M., Gollub, T., Wiegmann, M., Stein, B.: TIRA Integrated Research Architecture. In: Ferro, N., Peters, C. (eds.) Information Retrieval Evaluation in a Changing World. The Information Retrieval Series, Springer (Sep 2019) +10. Rieke, R.D., Sillars, M.O., Peterson, T.R.: Argumentation and critical decision making. Longman New York (1997) +11. Stab, C., Daxenberger, J., Stahlhut, C., Miller, T., Schiller, B., Tauchmann, C., Eger, S., Gurevych, I.: Argumentext: Searching for arguments in heterogeneous sources. In: Proceedings of the 2018 conference of the North American chapter of the association for computational linguistics: demonstrations. pp. 21-25 (2018) +12. Varior, R.R., Shuai, B., Lu, J., Xu, D., Wang, G.: A siamese long short-term memory architecture for human re-identification. In: European conference on computer vision. pp. 135-153. Springer (2016) +13. Wachsmuth, H., Naderi, N., Hou, Y., Bilu, Y., Prabhakaran, V., Thijm, T.A., Hirst, G., Stein, B.: Computational argumentation quality assessment in natural language. In: Proceedings of the 15th Conference of the European Chapter of the Association for Computational Linguistics: Volume 1, Long Papers. pp. 176-187 (2017) +14. Wachsmuth, H., Potthast, M., Al Khatib, K., Ajjour, Y., Puschmann, J., Qu, J., Dorsch, J., Morari, V., Bevendorff, J., Stein, B.: Building an argument search engine for the web. In: Proceedings of the 4th Workshop on Argument Mining. pp. 49-59 (2017) +15. Xiong, C., Dai, Z., Callan, J., Liu, Z., Power, R.: End-to-end neural ad-hoc ranking with kernel pooling. In: Proceedings of the 40th International ACM SIGIR conference on research and development in information retrieval. pp. 55-64 (2017) +16. Zamani, H., Dehghani, M., Croft, W.B., Learned-Miller, E., Kamps, J.: From neural reranking to neural ranking: Learning a sparse representation for inverted indexing. In: Proceedings of the 27th ACM International Conference on Information and Knowledge Management. pp. 497-506 (2018) \ No newline at end of file diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/tables/page007-table01.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/tables/page007-table01.png new file mode 100644 index 0000000..492136a Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/tables/page007-table01.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/tables/page011-table01.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/tables/page011-table01.png new file mode 100644 index 0000000..1891eb8 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/tables/page011-table01.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/tables/table-01.txt.md b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/tables/table-01.txt.md new file mode 100644 index 0000000..dbdbec5 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/tables/table-01.txt.md @@ -0,0 +1,4 @@ +| Model | MRR MAP nDCG | +|---|---| +| GRU | 28.4 24.1 38.05 | +| KNRM | 84.35 72.64 80.24 | diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/tables/table-02.txt.md b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/tables/table-02.txt.md new file mode 100644 index 0000000..18ffee3 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/172/tables/table-02.txt.md @@ -0,0 +1,6 @@ +| GRU | x | +|---|---| +| DRMM | x | +| KNRM | 68.4 | +| CKNRM | x | +| SNRM | x | diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/174/paper.pdf b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/174/paper.pdf new file mode 100644 index 0000000..755bcd0 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/174/paper.pdf differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/174/paper.txt.md b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/174/paper.txt.md new file mode 100644 index 0000000..57e9945 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/174/paper.txt.md @@ -0,0 +1,137 @@ +# Ranking Arguments by Combining Claim Similarity and Argument Quality Dimensions + +## Notebook for Touche: Argument Retrieval at CLEF 2020´ + +Lorik Dumani and Ralf Schenkel + +Trier University, Germany {dumani,schenkel}@uni-trier.de + +### Abstract In this paper we describe our submissions to the CLEF lab Touche,´ + +which addresses argument retrieval from a focused debate collection. Our approach consists of a two-step retrieval. Step one finds the most similar claims to a query. Step two ranks the directly tied premises by the count of their convincingness compared to other relevant premises, for which we aggregate the sum of three main argument quality dimensions. The final ranking consists of the product + +of the two components which are expressed as probabilities. + +## 1 Introduction + +Argumentation is required not only in political debates, where people try to convince others of certain standpoints, e.g., political views. They are also essential for personal decision making, e.g., which smartphone to buy. Since the emergence of well-equipped computers and the increasingly sophisticated NLP methods, computational argumentation has become a very popular field of research and seeks to help people to find good and strong arguments for their needs. In line with existing work, an argument is defined as a claim supported or attacked by at least one premise [13]. The claim is usually a controversial standpoint that should not be believed by a reader without further evidence + +(in form of premises). + +Touche [5,4] is the first lab on Argument Retrieval.´ 1 It follows the classical TRECstyle2 evaluation methodology and features two subtasks: + +1. Argument retrieval from a focused debate collection to support argumentative con- + +versations by providing justifications for the claims. + +2. Argument retrieval from a generic Web crawl to answer comparative questions with + +argumentative results and to support decision making. + +We participated in Task 1 and in this paper we provide a description of the implementation of our approach. Our submissions to the task were done under the team name Don + +Quixote. + +Copyright c 2020 for this paper by its authors. Use permitted under Creative Commons License Attribution 4.0 International (CC BY 4.0). CLEF 2020, 22-25 September 2020, Thessaloniki, Greece. + +1 https://events.webis.de/touche-20/. 2 https://trec.nist.gov/tracks.html. + +----- + +Task 1 aims at supporting users directly and finding arguments, e.g., to strengthen their stance or to form opinions about certain issues by "strong" arguments. Thus, besides general relevance of the argument to the topic, argument quality dimensions (see the work of Wachsmuth et al. [15] for a survey of work on argument quality) will be also evaluated. From the top-k results of all submissions, pools of answers will be formed that will be assessed by crowd-sourcing; evaluation of the submissions will then be done with nDCG [10]. The retrieved arguments will be evaluated by the following qualities: (1) whether an argumentative text is logically cogent, (2) whether it is rhetorically well-written, and (3) whether it contributes to the users' stance-building process (called utility). The Web provides innumerable documents that contain arguments. Especially in online portals controversial topics are discussed. Since basically anyone can participate in the discussion, we can assume that practically all relevant aspects are addressed there. Furthermore, as most of the participants are not experts, the arguments might be written in an understandable language. Hence, the official data basis is the dataset from Ajjour et al. [1,2], which consists of controversial discussions from debate portals3 and ChangeMyView, and on which the argument search engine args.me [14] also bases its arguments. The lab's participants can choose between the downloadable corpus and args' API.4 In our implementation we work with the downloadable dataset. Moreover, the lab provides 50 topics on different areas such as "abortion" or "gay marriage", for which the lab participants have to find strong arguments from the provided dataset. + +## 2 Our Approach: Concept and Implementation + +In this section we give a short overview of our approach. First, we introduce the general concept, then we discuss the preprocessing of the provided data as well as another dataset that we use to estimate the convincingness of premises. Finally, we show how we find "strong" arguments. + +### 2.1 Concept + +We follow the principles developed in [6,8], which we summarize here briefly. First the set of claims C = {c1, c2, . . . } of the collection is clustered such that claims with the same meaning are assigned to the same claim cluster, yielding Γ = {γ1, γ2, . . .} with γi, γj ⊆ C and γi ∩ γj = ∅ in an offline operation (see Section 2.2). Note that the claims in args.me are sometimes formulated as questions or topic titles. However, in the following we will only refer to claims. We precluster the claims because often, the same claim appears in different formulations in a large collection, and we want to consider all variants of the same claim at the same time. This is even more important for + +3 The arguments were extracted from the following debate portals: debatewise.org, + +idebate.org, debatepedia.org, and debate.org. 4 args' API: https://www.args.me/api-en.html. The downloadable dataset of args: https://zenodo.org/record/3734893#.Xw24QCgzaUk. + +----- + +premises, since premises with the same meaning, but different formulations are often used in many claims, but we want to retrieve that premise only once. We thus cluster the set of premises P = {p1, p2, . . .} and a set Π = {π1, π2, . . .} with πi, πj ⊆ P and *πi ∩ πj = ∅ of premise clusters is formed such that premises with the same meaning* are put into the same premise cluster. + +Now, given a query claim q, we apply a two-step retrieval approach. In the first step, our approach locates the claims in C that are most similar to q, following the observation that the more similar a claim is to a query, the more relevant are the premises of that claim to the query [7]. In the second step, we locate the claim clusters containing these claims, collect all premises related to a claim of one of these claim clusters, and finally determine the premise clusters to which these premises belong. For the ranking of premise clusters, we now apply a probabilistic ranking method. Thus, our goal is to compute P(πj |q), that is, the probability that πj is chosen as supporting or attacking premise cluster for q. We can calculate P(πj |q) by iterating over all premises in πj and aggregating their individual probabilities, i.e., P(πj |q) = P *P(p|q), where* + +*p∈πj* + +*P(p|q) is the probability that p is chosen as support or attack for q. P(p|q) is defined* by combining the two aforementioned steps, i.e., P(p|q) = P(c|q) · P(p|c). Formally, *P(c|q) denotes the probability that c is chosen as a similar claim to q. P(p|c) denotes* the probability that p is chosen as support (or attack) for c. + +In our previous work [6] we estimated P(p|c) exclusively via frequencies of premises. For our submissions to Task 1, we use a different approach [8] that also takes some dimensions of argument quality into account. We describe this approach in the following subsections. + +### 2.2 Preprocessing of the Provided Dataset + +Since the provided dataset by Ajjour et al. [1,2] originally consists of arguments with two components, that is, one claim with exactly one premise, we initially grouped all premises by their textually equal claim. Afterwards this grouping will become important because we calculate the convincingness of a premise in comparison to other premises of the same claim. Then the contextualized embeddings of both the claims and the premises were derived by using Sentence-BERT (SBERT) [12].5 For the clustering of claims as well as premises we followed the approach of our prior work [6] and implemented an agglomerative clustering applying Euclidean distance, the average linkage method, and a dynamic tree cut [11].6 + +5 The framework used for this is available on https://github.com/UKPLab/ sentence-transformers. The model we used for calculating the embeddings is "roberta-large-nli-stsb-mean-tokens", yielding embeddings of 1,024 dimensions each. 6 For the agglomerative clustering we used the scripting language R and the packages STATS and FASTCLUSTER. + +----- + +### 2.3 Including another Dataset to Estimate the Convincingness of Premises + +Wachsmuth et al. [15] provide a dataset in which three experts assessed 320 arguments with respect to 15 argument quality dimensions. The arguments are distributed over 32 issue-stance pairs, i.e., 16 topics with two polarities and 10 premises each. Among these 15 dimensions there are the three main dimensions: (1) logical quality in terms of the cogency or strength of an argument, (2) rhetorical quality in terms of the persuasive effect of an argument or argumentation + +(called effectiveness), and (3) dialectical quality in terms of the reasonableness of argumentation for resolving + +issues. + +We considered the mean assessment values for the three main dimensions cogency, rea*sonableness and effectiveness and integrated the idea of Habernal and Gurevych [9] by* deriving all combinations of (premise1, premise2) pairs with premises from the same issue-stance pairs and labels "1" or "2", whereby the labels signal which premise has a higher score with respect to a dimension. Pairs with equal mean value were omitted. Then, for the two premises of each pair as well as the corresponding (issue,stance) pair, we derived their SBERT embeddings, processed them to vectors consisting of the embeddings of the two premises each with the pointwise sum, difference, and product to the embedding of the corresponding (issue,stance) pair, yielding a vector of 6,144 dimension per (premise1, premise2) pair.7 Then, we tested standard classifiers such as gradient boosting, logistic regression, or random forest with 32-fold-cross-validation and found that random forest performs best for cogency and effectiveness. For reasonableness Logistic Regression performed only slightly better. Using these best classifiers per dimension, we were able to precalculate the dimension convincing frequencies (DCFs) of the premises in the datset by Ajjour et al. [1,2]. Here, the DCF of a premise of a claim is calculated as the count how often the premise was better than the other premises belonging to the same claim in a cross comparison. + +Now, both claims and premises could be indexed in two separate inverted indexes with the cluster and DCF information. We used Apache Lucene to build the indexes.8 + +### 2.4 Finding Strong Arguments + +For each of the 50 topics which we regard as queries, we started by finding the most similar claims (result claims) using Divergence from Randomness (DFR) [3], because our previous work [7] implies that DFR is well suited for this task. Then all premises belonging to claims that are in the same cluster as the result claims were localized. The set of premises was then expanded with the set of premises in the same premise clusters, yielding the set of result premises. + +Before calculating the premise cluster scores, first the premises were ranked individually. The ranking of these consists of the two components (1) similarity of the query + +7 The input can be determined by the elementwise computed Cartesian product of the embeddings of the following three sets, in compliance with the below order. The difference is positive. *{premise1, premise2}, {+, -, ∗}, {(issue,stance)-pair}.* + +8 https://lucene.apache.org/. + +----- + +to the claim and (2) the sum of the three different DCFs per premise (see Section 2.3). Both (1) and (2) were normalized to have values between 0 and 1, allowing to use them like probabilities (P(p|c) in the description above). The cluster scores were determined by aggregating the scores of the individual premises of the same cluster. From each cluster, only one representative was selected; in our implementation this is the longest premise as we followed the intuition that a longer premise is also more specific and therefore may be better suited as a representative. + +As trec eval sorts documents by the score values and not by rank values, it is important to handle tied scores. Furthermore, it is the score (integer or floating point) that is relevant for the TREC evaluation in the ranking. Therefore, the representatives were sorted in descending order by cluster score, then by length to break ties, and alphabetically if also the length was the same. To reflect this in the ranking, of all representatives with the same initial score, the scores were increased by the smallest possible delta in Java (10-17) starting from the premise at the bottom. + +**Subsequent Adjustions We manually reviewed the results of our retrieval at a cutoff** + +value of 30 and found that premises with less than 30 characters were usually completely useless as they are too unspecific or nonsense, so we removed them from the results. + +## 3 Conclusion + +In this paper we outlined our contribution (team Don Quixote) to the CLEF lab Touche.´ First we cluster claims and premises in an offline operation by their meaning. For a given query, we then work with a two-step retrieval process that first finds all similar claims and then, using the clusters, finds the relevant premises. For the ranking, we then calculate (1) the similarity of claim and query, and (2) the frequency with which a premise is more convincing than other relevant premises with respect to the three main argument quality dimensions cogency, reasonableness, and effectiveness. Describing (1) and (2) as probabilities, a ranking can be generated via their product. The code will be made available shortly. + +## Acknowledgements + +This work has been funded by the Deutsche Forschungsgemeinschaft (DFG) within the project ReCAP, Grant Number 375342983 - 2018-2020, as part of the Priority Program "Robust Argumentation Machines (RATIO)" (SPP-1999). + +## References + +1. Ajjour, Y., Wachsmuth, H., Kiesel, D., Riehmann, P., Fan, F., Castiglia, G., Adejoh, R., Frohlich, B., Stein, B.: Visualization of the topic space of argument search results in¨ args.me. In: Blanco, E., Lu, W. (eds.) EMNLP. pp. 60-65. Association for Computational Linguistics (2018). https://doi.org/10.18653/v1/d18-2011, https://doi.org/10.18653/v1/d18-2011 + +----- + +2. Ajjour, Y., Wachsmuth, H., Kiesel, J., Potthast, M., Hagen, M., Stein, B.: Data acquisition for argument search: The args.me corpus. In: Benzmuller, C., Stuckenschmidt, H. (eds.) KI. ¨ Lecture Notes in Computer Science, vol. 11793, pp. 48-59. Springer (2019). https://doi.org/10.1007/978-3-030-30179-8 4, https://doi.org/10.1007/978-3-030-30179-8\\\_4 +3. Amati, G., van Rijsbergen, C.J.: Probabilistic models of information retrieval based on measuring the divergence from randomness. ACM Transactions on Information Systems + +**20(4), 357-389 (2002). https://doi.org/10.1145/582415.582416** + +4. Bondarenko, A., Frobe, M., Beloucif, M., Gienapp, L., Ajjour, Y., Panchenko, A., Biemann,¨ C., Stein, B., Wachsmuth, H., Potthast, M., Hagen, M.: Overview of Touche 2020: ´ Argument Retrieval. In: Working Notes Papers of the CLEF 2020 Evaluation Labs (Sep 2020) +5. Bondarenko, A., Hagen, M., Potthast, M., Wachsmuth, H., Beloucif, M., Biemann, C., Panchenko, A., Stein, B.: Touche: First shared task on argument retrieval. In: ECIR. Lecture´ Notes in Computer Science, vol. 12036, pp. 517-523. Springer (2020). https://doi.org/10.1007/978-3-030-45442-5 67, https://doi.org/10.1007/978-3-030-45442-5\\\_67 +6. Dumani, L., Neumann, P.J., Schenkel, R.: A framework for argument retrieval - ranking argument clusters by frequency and specificity. In: ECIR. Lecture Notes in Computer Science, vol. 12035, pp. 431-445. Springer (2020). https://doi.org/10.1007/978-3-030-45439-5 29, https://doi.org/10.1007/978-3-030-45439-5\\\_29 +7. Dumani, L., Schenkel, R.: A systematic comparison of methods for finding good premises for claims. In: SIGIR. pp. 957-960 (2019), https://doi.org/10.1145/3331184.3331282 +8. Dumani, L., Schenkel, R.: Quality-aware ranking of arguments. In: CIKM (2020), accepted +9. Habernal, I., Gurevych, I.: Which argument is more convincing? Analyzing and predicting convincingness of web arguments using bidirectional LSTM. In: ACL (2016), https://www.aclweb.org/anthology/P16-1150/ +10. Jarvelin, K., Kek¨ al¨ ainen, J.: Cumulated gain-based evaluation of IR techniques. ACM¨ Transactions on Information Systems 20(4), 422-446 (2002). https://doi.org/10.1145/582415.582418, http://doi.acm.org/10.1145/582415.582418 +11. Langfelder, P., Zhang, B., Horvath, S.: Dynamic tree cut: In-depth description, tests and applications (2009), https://horvath.genetics.ucla.edu/html/ CoexpressionNetwork/BranchCutting/Supplement.pdf +12. Reimers, N., Schiller, B., Beck, T., Daxenberger, J., Stab, C., Gurevych, I.: Classification and clustering of arguments with contextualized word embeddings. In: ACL. pp. 567-578 (2019), https://www.aclweb.org/anthology/P19-1054/ +13. Stede, M., Afantenos, S.D., Peldszus, A., Asher, N., Perret, J.: Parallel discourse annotations on a corpus of short texts. In: LREC (2016), http: //www.lrec-conf.org/proceedings/lrec2016/summaries/477.html +14. Wachsmuth, H., Potthast, M., Khatib, K.A., Ajjour, Y., Puschmann, J., Qu, J., Dorsch, J., Morari, V., Bevendorff, J., Stein, B.: Building an argument search engine for the web. In: ArgMining@EMNLP. pp. 49-59. Association for Computational Linguistics (2017). https://doi.org/10.18653/v1/w17-5106, https://doi.org/10.18653/v1/w17-5106 +15. Wachsmuth, H., Stein, B., Hirst, G., Prabhakaran, V., Bilu, Y., Hou, Y., Naderi, N., Alberdingk Thijm, T.: Computational argumentation quality assessment in natural language. In: EACL. pp. 176-187 (2017), https://aclweb.org/anthology/E17-1017/ \ No newline at end of file diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/paper.pdf b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/paper.pdf new file mode 100644 index 0000000..59ca290 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/paper.pdf differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/paper.txt.md b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/paper.txt.md new file mode 100644 index 0000000..ca898ba --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/paper.txt.md @@ -0,0 +1,227 @@ +# Creating an Argument Search Engine for Online Debates + +Notebook for the Touch´e Lab on Argument Retrieval at CLEF 2020 + +Maximilian Bundesmann1, Lukas Christ2, and Matthias Richter3 + +1 University of Leipzig, Germany mb74fawu@studserv.uni-leipzig.de 2 University of Leipzig, Germany lc85futa@studserv.uni-leipzig.de 3 University of Leipzig, Germany mr75syri@studserv.uni-leipzig.de + +Abstract. Consulting web search engines has become an everyday procedure for many internet users. One specific task that gained attention in recent work is the retrieval of arguments for controversial topics. Most of the preexisting difficulties that search engines have to face also apply for this task. However, certain challenges become even more important, such as providing an appropriate heterogeneity in the result set. We present an argument search engine for the argsme corpus. Our focus is on preprocessing the corpus while also addressing the heterogeneity problem and implementing a query expansion feature. Furthermore, we provide a + +brief evaluation of our retrieval results. + +Keywords: Information Retrieval · Argumentative Conversations · Online Debates · Argument Search + +## 1 Introduction + +Nowadays, search engines are used by everybody. Consulting a search engine is the easiest way to find desired information like today's weather, news articles or just any arbitrary image. However, there are still some problems for which modern search engines fail to deliver satisfying answers yet. One of these challenges is the search for arguments in large document collections, e.g. for debates such as "Are plastic bottles good?" or "Are speed limits wrong?". Search engines that find arguments for these kinds of queries could be classified as argument *search engines. There are already some solutions available, e.g. Args.me [22] or* + +ArgumenText [20]. This report is created in the context of the Touch´e shared task on argument + +retrieval [5]. Its goal is to develop an argument search engine that retrieves arguments from the argsme corpus [2], which provides almost 390,000 arguments + +Copyright c 2020 for this paper by its authors. Use permitted under Creative Commons License Attribution 4.0 International (CC BY 4.0). CLEF 2020, 22-25 September 2020, Thessaloniki, Greece. + +----- + +from over 55,000 online debates. We develop a search engine to find good arguments in the corpus. This report describes our approach and evaluates its performance. + +First of all in Section 2, we summarize the state of the art of information retrieval in the context of argument search engines. Section 3 introduces our search engine's architecture. Following, we give a short overview of the corpus, describe some necessary preprocessing steps and show the ideas behind all separate components. In Section 4 we present the results of the final evaluation. + +Argument search refers to collecting relevant premises and conclusions to a given topic that is usually of controversial nature. The goal of such a search engine is to provide the user with supported statements that help him to gather knowledge about his topic of interest and potentially assist his decision making. + +## 2 Related Work + +Previous work has been carried out that tackled various tasks of argument search, including automatically detecting evidence that support a given claim [19], determining argument relevance [17] or acquiring a corpus of arguments [1]. The latter one along with the work presented by Wachsmuth et al. [22] constitutes the basis for this work. + +2.1 Retrieval Models + +The heart of an argument search engine is a proper retrieval model. The challenge here is to find the best arguments w.r.t. a corpus and a free-text query. + +Several argument search engines are e.g. args.me [22] or ArgumenText [20]. These engines are based on the retrieval model Okapi BM25. Potthast et al. [17] performed a user study to evaluate different well-known retrieval models. In this examination Lucene's BM25, Terrier's implementations of DPH [3], DirichletLM [25], and TFIDF were considered. The retrieved arguments by the different retrieval models were rated by their relevance, rhetoric, logic, and dialectic quality. DPH proved to yield the best results overall. + +2.2 Preprocessing + +Predicting argument quality is a challenging task. Wachsmuth et al. [21] discuss the concept of argument quality. Wei et al. [24] rank argumentative reddit posts in order to find the most persuasive ones. Their approach is machine learningbased. The same applies for the approach proposed by Persing and Ng [16]. Potthast et al. [17] provide a subset of the argsme corpus in which arguments were manually annotated with a rating for the quality aspects defined by Wachsmuth et al. [21]. Furthermore, an overall quality rating was assigned to each argument. + +----- + +2.3 Query Expansion Several different approaches have been explored that aim to improve the recall for user queries [8]. Some examples are pseudo-relevance feedback (using terms from the top ranked documents) or interactive query refinement that requires the user to readjust his query. Another option is to use search query logs to obtain rewritings that users perform to improve their query terms with respect to their information need. However, some of these methods are out of the scope for this work, or require additional data. For instance, search query logs are unavailable for this task. Therefore, we focus on a few automatic query expansion (AQE) methods based on word embeddings. Diaz et al. [12] and Zuccon et al. [26], for instance, utilized such "model-based" approaches. + +2.4 Clustering The aim of clustering here is to guarantee a diverse result set in order to present the user a variety of different arguments. Carbonell and Goldstein [7] introduce Maximal Marginal Relevance (MMR), a measure that allows building a ranking incrementally. MMR balances the quality and heterogeneity of the result set. + +Another approach to build a diverse ranking incrementally is proposed by Kaptein et al. [15]. Deselaers et al. [11] describe a method to diversify image search results using a "novelty" measure. A problem in diversifying search results is that classic evaluation measures like (n)DCG do not take diversity of results into account. Clarke et al. [9] thus propose an alternative evaluation framework. + +## 3 Methods + +Our argument search engine's architecture is depicted in Figure 1. The central module is the Apache Lucene Core4 search library that realizes indexing and retrieval. Before indexing, the corpus is preprocessed. During preprocessing, quality ratings for all documents are computed. + +At run time, the user's queries are enriched by a query expansion module. Then, after retrieving a set of relevant documents via the Lucene Core, results are ranked considering both the scores obtained by the retrieval system and the quality ratings. Eventually, the last component can perform clustering on the top ranked results. + +3.1 Preprocessing The quality of the arguments contained in the corpus is heterogeneous. Some documents do not contain arguments at all. Therefore we aim to assign ratings to the documents indicating their argumentative quality. More formally, we create a mapping + +*q : D → [0, 1]* (1) + +4 https://lucene.apache.org/ + +----- + +Fig. 1. All components of the introduced pipeline can be changed independently. + +where D is the corpus as a set of documents. If q(d1) > q(d2), the argumentative quality of d1 is considered higher than that of d2. These ratings are then used in our retrieval model, as the user should only receive arguments with high quality. To compute them, we employ a machine learning approach. Argument quality is a rather elusive concept that can not be quantified directly. Wachsmuth et al. [21] break it down into three aspects: - Logical quality: are the premises acceptable and do they really imply the + +conclusion? - Rhetorical quality: is the argument formulated in a persuasive manner? - Dialectical quality: does the argument contribute to resolving the issue? Capturing the logical dimension of argument quality with computational features is a hard task. Solving it is beyond the scope of this project.The dialectical quality dimension is not available in our corpus either. Other than the corpus of reddit posts used by Wei et al. [24], the argsme corpus does not contain information about replies to a post or citations of a post. Thus, the only quality dimension we aim to quantify is rhetorical quality. To achieve this, we compute 22 features for each argument. Most of them can already be found in Wei et al. [24] and Persing and Ng [16]. In the following, the features are briefly described. Linguistic competence features aim at quantifying the argument's author's linguistic skills: average sentence length, word length, type/token ratio, number of *punctuation marks per sentence, different POS-Tags (whole text), conjunctives* *per sentence, modal verbs per sentence, emojis (Use of emojis might coincide* with rather colloquial language and lack of seriousness), non-stopwords ratio. Sources and Examples: claims are more persuasive when they are supported by examples and sources. The following rule-based features are intended to capture them: number of references per sentence, examples per sentence, URLs per *sentence, percentages per sentence, year specifications per sentence* + +----- + +Subjectivity, ad hominem and emotionality: Arguments are more persuasive when they are presented in an objective manner, without anecdotal evidence or attacking the opponent personally. We aim to quantify subjectivity and emotionality with the following features: - number of first person pl. pronouns per sentence indicate subjectivity. We do not count first person singular words. Persing and Ng [16] argue that objective arguments frequently start with phrases like "I think..." or "I believe...", too. - number of second person pronouns per sentence may indicate personal attacks - Sentiment Analysis is able to indicate high emotionality. We use VADER + +(Hutto et al. [14]). - hedge words/phrases per sentence: these phrases may indicate a more polite, indirect and differentiated formulation. We use a list5 to identify such phrases. - number of definite articles / number of articles: Persing and Ng [16] argue that a lack of definite articles often means a lack of specifity and objectivity - average concreteness: Brysbaert et al. [6] provide ratings for word concreteness obtained by crowd sourcing. This feature describes the average degree of abstractness/concreteness in the argument. - components of emotions: Words affect our emotions. According to Warriner + +et al. [23], there are three components of each emotion: + +- *valence, i.e. "pleasantness"([23]), ranges from "happy" to "unhappy"* +- *arousal is "the intensity of emotion provoked by a stimulus" ([6])* +- *dominance denotes "the degree of control exerted by a stimulus" ([23])* For each of these emotion components, Warriner et al. provide word ratings. We build three features: average valence of words in the argument, average arousal and average dominance. Before normalizing all features we filter out odd documents based on rules. To give an example, the average word length in an argument is expected to be between 2 and 16. Odd documents are assigned the rating 0.0. Such documents are typically spam or short meta-posts like e.g. "I accept", "Vote Pro" etc. As training data we use the Webis-ArgQuality-20 Corpus [13]. It contains about 1600 arguments from the argsme corpus. Furthermore, it provides ratings for all three argument quality dimensions as well as for combined/overall argument quality. These continuous ratings range from -4.0 (not an argument) to 4.0. We train several several machine learning models: Linear Regression, Decision Tree Regression and Support Vector Regression (SVR) with different kernels. For each type of model we train one instance on rhetorical quality and another instance on combined quality. Both instances' parameters are optimized via grid search. + +5 https://github.com/words/hedges + +----- + +All models perform rather poorly, confirming that argument quality prediction is a difficult problem. SVR with a quadratic kernel achieves the best results (MSE of 1.641 for rhetorical and 1.475 for combined quality). Moreover, we train an ensemble model (Linear Regression) using the predictions of all models as features. As expected, it outperforms all single models (MSE of 1.468 for rhetorical and 1.322 for combined quality). + +An interesting detail is that all models, even those trained on rhetorical quality, perform better in predicting combined argument quality than in predicting rhetorical quality. In other words, overall quality seems to be easier to grasp than rhetorical quality, at least with our approach. This hypothesis is statistically significant for p < 0.01. One explanation may be that some of our features also capture aspects of dialectical and logical quality: For example, providing sources to support a claim could indicate logical correctness. Features related to subjectivity and emotionality might at least be able to suggest low dialectical quality, as a very emotional and/or subjective post is often unlikely to contribute to resolving an issue. + +Finally, to obtain the desired quality function q : D → [0, 1], we let the trained models predict the combined quality of every argument in the argsme corpus. We compute the predictions of the best single model (quadratic SVR trained on combined quality) and the ensemble model, leading to two candidates qsvr, qens for q. Figure 2 shows the distributions of the ratings generated by both models. + +Fig. 2. Distributions of ratings produced by ensemble classifier (qens) and SVR classifier (qsvr) + +We choose qsvr for q, even though the SVR model's MSE is higher than that of the ensemble method. The main reason for this decision is that there are almost no "bad" arguments according to qens, which is certainly inaccurate. + +----- + +3.2 Retrieval Model Like the implementation of args.me we decide to use Apache Lucene for the indexing and retrieval tasks. We index the extended corpus which is generated during preprocessing. In the first step of query processing, stopwords are removed from the query. Before results are retrieved, the query is extended using additional query expansion methods. For the ranking, we implement different methods. As a baseline, we use Lucene's BM25 implementation. Furthermore, we extend the Lucene search core with an implementation of the DPH concept [4]. These retrieval methods do not consider the quality ratings q of our extended corpus. To gain a profit from q we perform a reranking. The scoring function for a document d is given by: + +*score(d) = α · s 0* (d) + (1 - α) · q(d) (2) where s *0* is the normalized score of the retrieval model. A reasonable value of α (α ∈ [0, 1]) can be determined empirically. Initially, we set α = 0.5. + +3.3 Query Expansion A query is a short representation of the user's information need. However, these few words may not be sufficient to encompass the entire concept that the user wants to express. This can lead to highly relevant documents not being found by the retrieval system due to vocabulary missmatch. That is, the user may choose terms for his query that do not appear in a relevant document. To mitigate this gap, automatic query expansion methods can be used. In this section we briefly describe the components of AQE and our implementation. For our query expansion component we decided to use one simple baseline approach and two more sophisticated concepts. As baseline, we employ WordNet to fetch semantically similar words for each individual query term. This method can not grasp the concept of the entire query as one unit. However, as many of the queries provided for the shared task only consist of few terms, such as "speed limit" or "nuclear weapons", this simple AQE method can potentially provide a useful enhancement. The other two expansion procedures both rely on word embeddings. We use fastText6 to obtain vector representations from the argsme corpus. We combine these locally trained representations with pre-trained embeddings offered by fastText. Then, we adapt a query expansion method as proposed by Diaz et al. [12]. This model-based expansion procedure searches the word embeddings for semantically similar terms in order to estimate an alternative to the original query by interpolating the query language model pq with that of the expansion language pq+ as follows: + +*p* 1 (w) = λpq(w) + (1 - λ)pq+ (w) (3) + +*q* + +6 https://fasttext.cc/ + +----- + +All newly found terms are then weighted and the best ones (matching the modeled language) are selected to augment the query. Even though the work presented by Zuccon et al. [26] does not directly focus on AQE, we also use their insights to realize another expansion method. They investigate different ways to estimate translation probabilities for terms that belong to the same language model. Similarly, our goal for AQE is to find words *w that are likely to be "translations" of the initial query terms:* + +*pt(w|q) = Σu∈qpt(w|u)p(u|q)* (4) + +where pt(w|u) describes the probability of translating term u into w which can be approximated by a normalized cosine similarity. Naturally, both expansion techniques operate on each query as a whole to incorporate their relatedness. Eventually, the expansion terms and their respective weights are returned to our search core. Note that for the current implementation only one expansion method is used at a time. + +3.4 Clustering and Reranking A more experimental component of our search engine is the clustering/reranking module. In retrieving arguments, not only the argumentative quality of the returned results is important. Another aspect of an argument search engine's utility is the heterogeneity of the returned arguments. In every use case, the user benefits from receiving a wide variety of semantically different arguments. A problem of the argsme corpus is that an argumentative document usually contains more than one argument. Nevertheless, documents may often be semantically similar. Moreover, optimizing heterogeneity can conflict with optimizing quality. Both goals need to be balanced. As the Touch`e task is evaluated using nDCG, we first make sure that our results are of high quality (w.r.t to the query and the argumentation quality). Then, the top 8 results are clustered and reranked in order to diversify the top results. Semantic clustering is implemented using Latent Semantic Analysis (Deerwester et al. [10]) for 3 topics. This provides a vector of size 3 for each of the top 8 documents. Now, the distance dist(d1, d2) = 1-SIM(d1, d2), i.e. dissimilarity between two documents d1 and d2 can be described in terms of the 3-dimensional vectors generated by LSA. In the following, let R be the ranking and R[i] the document with rank *i in R. Similar to Deselaers et al. [11], we employ a notion of a document's* novelty. Novelty of a document R[i] is related to R[i]0s predecessors in the ranking *R[1]...R[i - 1]:* + +*Nov(R[i]) := Σ i-1 dist(R[i], R[i - k])* (5) + +*k=1 k* + +We weight the dissimilarity depending on the number of ranks between R[j] and *R[j - k]: documents should not be similar to their immediate predecessor.* + +----- + +Based on novelties, we define a measure for R's diversity/heterogeneity: + +*|R|* 1 + +*heterogeneity(R) := Σ Nov(R[j])* (6) + +*j=2 j* + +The likelihood that a user actually looks at a document d decreases with d's rank. Because of that, our heterogeneity measure weights each document's novelty depending on its rank. Next, heterogenity of a ranking R needs to be balanced with R's quality. To achieve this, we compute a reranking R0 of R that maximizes + +*γ ∗ quality(R 0* ) + (1 - γ) ∗ heterogeneity(R *0* ) (7) We use nDCG with our retrieval model's ratings to compute quality(R0). Note that the problem of finding an optimal R0 can be framed as Mixed Integer Program. However, since we restrict ourselves to reranking only the top 8 documents, we find an optimal solution using brute force. Table 1 shows the effect of our reranking on a dummy corpus. + +Table 1. Examples for reranking on a dummy subset of the argsme corpus, for different values of γ in Equation 7. In Brackets the dummy quality value for each "argument". + +| Initial ranking (γ = 1.) | | | +|---|---|---| +| Vote Con! (1.0) | | | +| Vote Con (0.875) | | | +| Vote for Con. (0.75) | | | +| Vote PRO. (0.625) | | | +| vote for pro (0.5) | | | +| Extend my arguments. (0.375) | | | +| Please extend all arguments (0.25) | Vote PRO. (0.625) | | +| extend all arguments (0.125) | | | + +*γ = 0.5* Vote Con! (1.0) Vote for Con. (0.75) Extend my arg... (0.375) vote for pro (0.5) Vote Con (0.875) Please extend ... (0.25) vote for pro (0.5) extend all arg... (0.125) + +*γ = 0* Vote PRO. (0.625) Please extend ... (0.25) extend all arg... (0.125) Vote Con (0.875) Vote for Con. (0.75) Extend my arg... (0.375) Vote Con! (1.0) + +--- + +The hyperparameter γ ∈ [0, 1] in Equation 7 could be set by the user. Alternatively, γ could be further investigated in order find a reasonable value. This is beyond the scope of our project. For the evaluation, we turn off the clustering component (i.e. set γ to 1), because nDCG does not consider heterogeneity. + +## 4 Evaluation and Results + +For the final evaluation of our system, we decide to use the combination of DPH and baseline query expansion. Additionally, we augment the scoring function with our quality ratings as described in Equation 2 using α = 0.5. The clustering component is not used, since it can not be expected to have a positive impact on nDCG scores, as pointed out in Section 3.4. Among the various retrieval models, DPH should show the best performance according to the findings of Potthast et al. [17]. Moreover, some quick experiments with manually labelled test data + +----- + +have shown that among our query expansion methods, the baseline expansion achieves the most satisfying results. + +The result of the final run which was evaluated via tira.io [18] reaches a sound nDCG@5 of 0.804. For the older version of the corpus, this run was the best performing among all participants7 , indicating that the employed combination could be suitable to perform argument search tasks. We did not submit a run for the more recent corpus version. + +## 5 Conclusion and Outlook + +We implemented an argument search engine for the argsme corpus. The results, however, are not very convincing yet. We suppose they could be improved in future work, considering the following aspects. As the search engine proved to benefit from our argument quality ratings, these ratings could be further investigated. More sophisticated features and models could be tested. What is more, the weighting of the ratings in our retrieval model, i.e. the hyperparameter α in Equation 2, could be optimized. One of the major downsides of our approach is that it does not analyse the semantics of potentially relevant documents. Thus, the precision is often rather low. Future work could tackle this issue. A closer investigation of the query expansion component (e.g. investigating more queries) would probably improve our search engine's results, too. We implement a reranking component to diversify the top-ranked results. However, we were not able to evaluate its quality within the scope of this work. What is more, the reranking component is only implemented in a proof-of-concept style, limited to the top 8 documents. + +To conclude, we aimed to address the complex problem of argument retrieval using several different methods. There is much space for extending and enhancing our approach in order to improve its performance. + +## References + +1. Ajjour, Y., Wachsmuth, H., Kiesel, J., Potthast, M., Hagen, M., Stein, B.: Data acquisition for argument search: The args. me corpus. In: Joint German/Austrian Conference on Artificial Intelligence (K¨unstliche Intelligenz). pp. 48-59. Springer (2019) +2. Ajjour, Y., Wachsmuth, H., Kiesel, J., Potthast, M., Hagen, M., Stein, B.: Data Acquisition for Argument Search: The args.me corpus. In: Benzm¨uller, C., Stuckenschmidt, H. (eds.) 42nd German Conference on Artificial Intelligence (KI 2019). pp. 48-59. Springer (Sep 2019). https://doi.org/10.1007/978-3-030-30179-8 4 +3. Amati, G.: Frequentist and bayesian approach to information retrieval. In: European Conference on Information Retrieval. pp. 13-24. Springer (2006) +4. Amati, G.: Frequentist and bayesian approach to information retrieval. In: European Conference on Information Retrieval. pp. 13-24. Springer (2006) 7 We were automatically assigned the team name Weiss Schnee + +----- + +5. Bondarenko, A., Fr¨obe, M., Beloucif, M., Gienapp, L., Ajjour, Y., Panchenko, A., Biemann, C., Stein, B., Wachsmuth, H., Potthast, M., Hagen, M.: Overview of Touch´e 2020: Argument Retrieval. In: Working Notes Papers of the CLEF 2020 Evaluation Labs (Sep 2020) +6. Brysbaert, M., Warriner, A.B., Kuperman, V.: Concreteness ratings for 40 thousand generally known english word lemmas. Behavior research methods 46(3), 904-911 (2014) +7. Carbonell, J., Goldstein, J.: The use of mmr, diversity-based reranking for reordering documents and producing summaries. In: Proceedings of the 21st annual international ACM SIGIR conference on Research and development in information retrieval. pp. 335-336 (1998) +8. Carpineto, C., Romano, G.: A survey of automatic query expansion in information retrieval. Acm Computing Surveys (CSUR) 44(1), 1-50 (2012) +9. Clarke, C.L., Kolla, M., Cormack, G.V., Vechtomova, O., Ashkan, A., B¨uttcher, S., MacKinnon, I.: Novelty and diversity in information retrieval evaluation. In: Proceedings of the 31st annual international ACM SIGIR conference on Research and development in information retrieval. pp. 659-666 (2008) +10. Deerwester, S., Dumais, S.T., Furnas, G.W., Landauer, T.K., Harshman, R.: Indexing by latent semantic analysis. Journal of the American society for information science 41(6), 391-407 (1990) +11. Deselaers, T., Gass, T., Dreuw, P., Ney, H.: Jointly optimising relevance and diversity in image retrieval. In: Proceedings of the ACM international conference on image and video retrieval. pp. 1-8 (2009) +12. Diaz, F., Mitra, B., Craswell, N.: Query expansion with locally-trained word embeddings. arXiv preprint arXiv:1605.07891 (2016) +13. Gienapp, L., Stein, B., Hagen, M., Potthast, M.: Efficient Pairwise Annotation of Argument Quality. In: 58th Annual Meeting of the Association for Computational Linguistics (ACL 2020). pp. 5772-5781. Association for Computational Linguistics, Online (Jul 2020), https://www.aclweb.org/anthology/2020.acl-main.511 +14. Hutto, C.J., Gilbert, E.: Vader: A parsimonious rule-based model for sentiment analysis of social media text. In: Eighth international AAAI conference on weblogs and social media (2014) +15. Kaptein, R., Koolen, M., Kamps, J.: Result diversity and entity ranking experiments: Anchors, links, text and wikipedia. Tech. rep., AMSTERDAM UNIV (NETHERLANDS) INTELLIGENT SYSTEMS LAB AMSTERDAM (2009) +16. Persing, I., Ng, V.: Why can't you convince me? modeling weaknesses in unpersuasive arguments. In: IJCAI. pp. 4082-4088 (2017) + +| 17. Potthast, mann, N., | M., | Gienapp, | L., Wachsmuth, H., Stein, B., | Euchner, | F., Hagen, | Heilenk¨otter, | N., M.: Argument Search: | Weid- | +|---|---|---|---|---|---|---|---|---| +| Assessing | | Argument | Relevance. | In: | 42nd | International | | ACM Con- | +| ference GIR 2019). | on | Research ACM (Jul 2019). | and | Development | in | Information https://doi.org/10.1145/3331184.3331327, | Retrieval | (SI- | + +http://doi.acm.org/10.1145/3331184.3331327 + +18. Potthast, M., Gollub, T., Wiegmann, M., Stein, B.: TIRA Integrated Research Architecture. In: Ferro, N., Peters, C. (eds.) Information Retrieval Evaluation in a Changing World. The Information Retrieval Series, Springer (Sep 2019). https://doi.org/10.1007/978-3-030-22948-1 5 +19. Rinott, R., Dankin, L., Alzate Perez, C., Khapra, M.M., Aharoni, E., Slonim, N.: Show me your evidence - an automatic method for context dependent evidence detection. In: Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing. pp. 440-450. Association for Computational + +----- + +Linguistics, Lisbon, Portugal (Sep 2015). https://doi.org/10.18653/v1/D15-1050, https://www.aclweb.org/anthology/D15-1050 + +20. Stab, C., Daxenberger, J., Stahlhut, C., Miller, T., Schiller, B., Tauchmann, C., Eger, S., Gurevych, I.: Argumentext: Searching for arguments in heterogeneous sources. In: Proceedings of the 2018 conference of the North American chapter of the association for computational linguistics: demonstrations. pp. 21-25 (2018) +21. Wachsmuth, H., Naderi, N., Hou, Y., Bilu, Y., Prabhakaran, V., Thijm, T.A., Hirst, G., Stein, B.: Computational argumentation quality assessment in natural language. In: Proceedings of the 15th Conference of the European Chapter of the Association for Computational Linguistics: Volume 1, Long Papers. pp. 176-187 + +(2017) + +22. Wachsmuth, H., Potthast, M., Al-Khatib, K., Ajjour, Y., Puschmann, J., Qu, J., Dorsch, J., Morari, V., Bevendorff, J., Stein, B.: Building an argument search engine for the web pp. 49-59 (Sep 2017). https://doi.org/10.18653/v1/W17-5106, https://www.aclweb.org/anthology/W17-5106 +23. Warriner, A.B., Kuperman, V., Brysbaert, M.: Norms of valence, arousal, and dominance for 13,915 english lemmas. Behavior research methods 45(4), 1191- 1207 (2013) +24. Wei, Z., Liu, Y., Li, Y.: Is this post persuasive? ranking argumentative comments in online forum. In: Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers). pp. 195-200 (2016) +25. Zhai, C., Lafferty, J.: A study of smoothing methods for language models applied to ad hoc information retrieval. In: Proceedings of the 24th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval. Association for Computing Machinery, New York, NY, USA (2001). https://doi.org/10.1145/383952.384019, https://doi.org/10.1145/383952.384019 +26. Zuccon, G., Koopman, B., Bruza, P., Azzopardi, L.: Integrating and evaluating neural word embeddings in information retrieval. In: Proceedings of the 20th Australasian document computing symposium. pp. 1-8 (2015) \ No newline at end of file diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table01.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table01.png new file mode 100644 index 0000000..106f8cc Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table01.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table02.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table02.png new file mode 100644 index 0000000..548e940 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table02.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table03.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table03.png new file mode 100644 index 0000000..7f579b6 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table03.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table04.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table04.png new file mode 100644 index 0000000..125c0d3 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table04.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table05.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table05.png new file mode 100644 index 0000000..08ea5cf Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table05.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table06.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table06.png new file mode 100644 index 0000000..bf55332 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table06.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table07.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table07.png new file mode 100644 index 0000000..84b792c Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table07.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table08.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table08.png new file mode 100644 index 0000000..2008add Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table08.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table09.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table09.png new file mode 100644 index 0000000..b189aef Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/page009-table09.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/table-01.txt.md b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/table-01.txt.md new file mode 100644 index 0000000..53d5eb5 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/table-01.txt.md @@ -0,0 +1,10 @@ +| Initial ranking (γ = 1.) | | | +|---|---|---| +| Vote Con! (1.0) | | | +| Vote Con (0.875) | | | +| Vote for Con. (0.75) | | | +| Vote PRO. (0.625) | | | +| vote for pro (0.5) | | | +| Extend my arguments. (0.375) | | | +| Please extend all arguments (0.25) | Vote PRO. (0.625) | | +| extend all arguments (0.125) | | | diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/table-02.txt.md b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/table-02.txt.md new file mode 100644 index 0000000..af5d3e3 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/182/tables/table-02.txt.md @@ -0,0 +1,4 @@ +| 17. Potthast, mann, N., | M., | Gienapp, | L., Wachsmuth, H., Stein, B., | Euchner, | F., Hagen, | Heilenk¨otter, | N., M.: Argument Search: | Weid- | +|---|---|---|---|---|---|---|---|---| +| Assessing | | Argument | Relevance. | In: | 42nd | International | | ACM Con- | +| ference GIR 2019). | on | Research ACM (Jul 2019). | and | Development | in | Information https://doi.org/10.1145/3331184.3331327, | Retrieval | (SI- | diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/paper.pdf b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/paper.pdf new file mode 100644 index 0000000..387d5d8 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/paper.pdf differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/paper.txt.md b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/paper.txt.md new file mode 100644 index 0000000..e0061a0 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/paper.txt.md @@ -0,0 +1,156 @@ +# SentArg: A Hybrid Doc2Vec/DPH Model with Sentiment Analysis Refinement + +## Notebook for Touche: Argument Retrieval at CLEF 2020´ + +Christian Staudte1 and Lucas Lange2 + +1 Leipzig University, Germany + +cs47rica@studserv.uni-leipzig.de + +2 Leipzig University, Germany + +ll95wyqa@studserv.uni-leipzig.de + +**Abstract In this work we explore the yet untested inclusion of sentiment analysis** + +in the argument ranking process. By utilizing a word embedding model we create document embeddings for all queries and arguments. These are compared with each other to calculate top-N argument context scores for each query. We also calculate top-N DPH scores with the Terrier Framework. This way, each query receives two lists of top-N arguments. Afterwards we form an intersection of both argument lists and sort the result by the DPH scores. To further increase the ranking quality, we sort the final arguments of each query by sentiment values. Our findings ultimately imply that rewarding neutral sentiments can decrease the quality of the retrieval outcome. + +## 1 Introduction + +In this notebook we propose our SentArg model for the first task of Touche 2020: 1st ´ *Shared Task on Argument Retrieval3* [6]. The task is performed on the args.me cor*pus [1]. The source code is available on our public GitHub repository4* . SentArg is based on the Dual Embedding Space Model by Mitra et al. [11], although we replace BM25 with DPH [2] and adjust their scoring function. For further refinement we use sentiment analysis to classify the emotional level (positive, neutral, negative) of every argument utilizing the Google Cloud Natural Language API5 . Analysing the author's attitude, we thus re-rank arguments. Our main objective is to find out whether sentiments can increase the quality of an argument ranking. The remainder of our notebook is organized as follows. Section 2 outlines all related works we used as theoretical and practical foundation. In Section 3 we describe our SentArg model configuration. Our evaluation as well as experiments are then presented in Section 4. Section 5 concludes our notebook. + +Copyright c 2020 for this paper by its authors. Use permitted under Creative Commons License Attribution 4.0 International (CC BY 4.0). CLEF 2020, 22-25 September 2020, Thessaloniki, Greece. + +3 https://touche.webis.de 4 https://github.com/luckyos-code/ArgU 5 https://cloud.google.com/natural-language/ + +----- + +## 2 Related Work + +Words used in a mathematical model must be mapped onto mathematical objects. One*hot encodings are particularly well-known mappings for this: Each word is represented* by an element = 1 in vector -→v ∈ {0, 1} *|V ocab|* , whereas all other elements are 0. However, these representations scale with vocabulary size and are therefore extremely sparse. Further, semantic relations between words are not modeled [7]. To overcome these limitations, Mikolov et al. [10] introduce two neural network models: Continuous *Bag-of-Words (CBOW) and Skip-Gram (SG)6* . Both architectures typically apply one input-, one hidden-, and one output-layer. For any given word w, its left- and right-sided context serves as input for a CBOW model, which tries to predict w. After training, weight matrix WIN ∈ RM×N , with N being the embedding's vector size, contains all available word embeddings. Words that are semantically similar also occur in similar contexts, hence cosine similarities of their embeddings are close to 1. + +Regarding argument detection, similar contexts between queries and arguments may not be enough. To further restrict argument result sets, Mitra et al. [11] suggest a Dual Embedding Space Model. They calculate BM25 scores and word embedding similarities for any query Q regarding all arguments. Afterwards they form a weighted sum of both scores, which acts as argument relevance score. One special feature is the usage of CBOW's WOUT matrix. By calculating the cosine similarity of two word embeddings, each from a different weight matrix, co-occuring words reach scores close to 1. In their final results, IN-OUT-similarities provide the best results. + +Regarding typical information retrieval models, Potthast et al. [13] compare BM25, TFIDF, DirichletLM and DPH on more than 300.000 arguments (documents), containing 40 topics. Assessors evaluated different query results and analyzed various aspects of found arguments, such as relevance, as well as rhetorical, logical and dialectical quality. Their evaluation shows that DitrichletLM and DPH are superior to BM25 and TFIDF. With these results in mind, we decide to replace BM25 in the Dual Embedding Space Model with DPH. + +The goal of sentiment analysis is to identify the emotional character of a given text. This consideration is especially interesting in the case of arguments because research shows that arguments are fundamentally emotional [8,15]. The idea to include emotions in our ranking is based on the hypothesis that an emotional attachment to a topic can indicate involvement and thus stronger argumentation, which is backed by several studies [4,8,15,16]. We therefore expect a ranking in favor of emotional arguments to perform better than one favoring neutrality or one without sentiment analysis. + +A survey conducted by Lawrence and Reed [9] shows that works on sentiment analysis in argument retrieval mainly focus on opinion mining, i.e. guessing the stance of an argument. Therefore they do not use an already annotated corpus, rather they instead aim to create one. Transitioning sentiment analysis from retrieving to ranking arguments is a new approach provided through our model. + +6 We primarily focus on CBOW in this notebook, so please refer to the referenced paper [10] + +for more information on SG. + +----- + +## 3 Argument Retrieval Model + +Our model features three steps: 1. Pre-processing; 2. Training of word embeddings and DPH to calculate argument and query similarities; 3. Sentiment analysis to re-sort arguments. Concerning Google's sentiment analysis API, we first remove noise and formatting errors in as many arguments as possible. We manually define the following rule set: + +### - Remove URLs and square brackets with their content - Remove some punctuation {∼, #, §, &, @, =, *} + +### - Reformat punctuation with correct spacing - Replace identical letters in a row (>2) with one representative: helloooooo → hello + +Short arguments often express approval or disapproval with previous arguments, hence we delete arguments containing less than 26 words. For our dual embedding's CBOW model we replace/add the following rules: + +### - Numbers and URLs are replaced by tags and - Remove all punctuation - Remove stop words + +We keep upper- and lower-cases to reduce word ambiguities7 and train the CBOW model on all given arguments8 . However, this gives rise to problems concerning rare query terms: "E-Cigarettes" occures so rarely that no word embedding is trained. In this case we try different combinations of upper- and lower-case to find appropriate (and possibly multiple) word embeddings, which are all taken into account. After preprocessing and training a CBOW model, we compute document vectors for each argument as proposed by [11]: + +1 X *aj* *A = ·* (1) + +*|A| ||aj ||* + +*aj∈A* + +Each aj represents the word embedding for word j. Afterwards we calculate similarities between a query and all arguments: + +*DESM(Q, A) = 1* X *q i A* + +- (2) *|Q| ||q T || · ||A||* + +*qi∈Q i* + +DPH is the second relevant component we model with the help of the Terrier framework [12]. Contrary to the original dual embedding architecture, we do not calculate a weighted sum of scores. Mitra et al. [11] state that word embeddings are helpful for finding appropriate contexts, whereas BM25 is better at finding concrete details given any query. We therefore run DESM and DPH in parallel, both select the top 1.000 arguments.9 Then, we form an intersection of both sets and sort every argument by its DPH score. Only context relevant scores thus have influence on the result set. + +7 For example "US" 6= "us" + +8 Configuration: vector size = 300; window size = 3; min word count = 5 9 A manually selected and adjusted parameter. + +----- + +Sentiment analysis of the Google Cloud Natural Language API was created for a wide range of applications and is therefore most suitable for discussions in the data set. For every argument we send a request to the API, which in return provides us with a sentiment value. This value represents the direction of emotion on a scale from -1 to 1, with -1 and 1 being the strongest. Values between -0.1 and 0.1 represent neutral (N) sentiments, while values lower than -0.1 and greater than 0.1 express negative (-) and positive (+) sentiments, respectively. + +Obtained sentiments are combined with DPH scores to compute the final ranking. To confirm our expectations we introduce two variants: (i) We encourage emotional arguments (values closer to -1 and 1); (ii) We encourage neutral arguments (values closer to 0): + +*DP H∗ |SA|* + +*Q,A = DP HQ,A ± DP HQ,A ·* (3) + +2 *SA refers to the sentiment value for argument A. Adding the weighted DPH refers to* variant (i) and subtracting it to variant (ii). + +## 4 Experiments and Evaluation + +Since opinion mining states that sentiment analysis is capable of determining the stances of arguments [3,9], we tested if information is gained from sentiment analysis by comparing each argument's stance with its sentiment value. In the dataset, each stance is stored as Boolean (pro and con), neutral stances are therefore not given. We enriched these stances by defining a sentiment value range that includes neutral (N), positive (+) and negative (-) arguments. Table 1 displays the distribution of sentiment values compared to annotated stances. As can be seen, most arguments (57%) are neutral, while 11% are positive and 32% are negative. However, positive and negative arguments do not match the dataset's stance distribution of 53% pro and 47% con10 . Further, every sentiment class contains at least 42% candidates that are either pro or con. In spite of the opinion mining hypothesis, arguments for every sentiment class are nearly evenly distributed and no real correlation can be seen. In conclusion, by utilizing sentiment values we can gain new perspectives on the arguments. + +Part of the task was the submission and evaluation of our model on the TIRA platform [14].11 In the following, we refer to a new set of qrels (query relevances). To identify the influence of our sentiment strategy and model architecture on the retrieval, we evaluate six different runs representing the different variants (see Section 3): R0 No sentiment analysis, RE Emotional is better [cp. (i)], RN Neutral is better [cp. (ii)] and the two embedding types (IN-IN, IN-OUT). The results are shown in Table 2. RN delivers the lowest scores for both embedding types and all nDCG-measures. That means subtracting the absolute sentiment value from the DPH score penalizes arguments, which are relevant. The assumption would be that adding sentiment values can reward more relevant arguments. As can be seen in Table 2, this is partly true: For IN-OUT, RE + +10 These numbers are not mentioned in Table 1, even though the distribution for neutral sentiments looks this way. + +11 https://www.tira.io ; Group: ir-lab-ul-t1-detroitnitz + +----- + +reaches slightly better results than R0. But R0 moderately outmatches most scores of *RE when it comes to IN-IN. Emotional arguments seem to be preferred by the DPH* model at default, thus scores from runs R0 and RE are close to each other. In sum however, the overall top nDCG@X scores are reached by the configuration [RE, IN-OUT]. + +**Table 1. Comparison of the sentiment (Sent.) val-** + +ues (N, +, -) of arguments and their stances (Pro, Con). + +| | Argument's share | Stance in | % per share | +|---|---|---|---| +| Sent. | in number (%) | Pro | Con | +| N | 169,152 (57%) | 53% | 47% | +| + | 33,406 (11%) | 58% | 42% | +| - | 94,460 (32%) | 50% | 50% | + +**Table 2. Evaluation for different configurations: Embedding types and sentiments' influence.** + +--- + +| | | | IN-OUT | | | | IN-IN | | +|---|---|---|---|---|---|---|---|---| +| | nDCG | nDCG@5 | nDCG@10 | QrelCov@10 | nDCG | nDCG@5 | nDCG@10 | QrelCov@10 | +| R0 | 0.365 | 0.649 | 0.553 | 6.24 | 0.390 | 0.635 | 0.538 | 6.06 | +| RE | 0.369 | 0.699 | 0.559 | 6.24 | 0.385 | 0.625 | 0.528 | 5.96 | +| RN | 0.337 | 0.517 | 0.456 | 5.18 | 0.359 | 0.500 | 0.437 | 4.94 | + +--- + +## 5 Conclusion + +By (1) combining IN-OUT argument embeddings and DPH to retrieve relevant arguments and (2) sorting arguments in regards to their sentiment value we could learn two things: Not only can IN-OUT embeddings improve the context space for a queries argument list, but also can rewarding neutral arguments in a final ranking reduce the quality of a retrieval. Our empirical findings match the expectations set forth in our analysis of existing literature, that more emotional arguments are more relevant than neutral ones. That means prioritizing arguments with high sentiment values can have a positive influence on the relevance ranking. + +Future work may benefit from replacing the CBOW model with a FastText [5] architecture, which generates good results especially for noisy data. Further, static parameters and the influence equation for sentiments can be optimized. This way different sentiment values could have a more diverse influence on the final score. + +----- + +## References + +1. Ajjour, Y., Wachsmuth, H., Kiesel, J., Potthast, M., Hagen, M., Stein, B.: Data Acquisition for Argument Search: The args.me corpus. In: Benzmuller, C., Stuckenschmidt, H. (eds.) ¨ 42nd German Conference on Artificial Intelligence (KI 2019). pp. 48-59. Springer (Sep 2019). https://doi.org/10.1007/978-3-030-30179-8 4 +2. Amati, G.: Frequentist and bayesian approach to information retrieval. pp. 13-24 (01 1970). https://doi.org/10.1007/11735106 3 +3. Bakshi, R.K., Kaur, N., Kaur, R., Kaur, G.: Opinion mining and sentiment analysis. In: 2016 3rd International Conference on Computing for Sustainable Global Development (INDIACom). pp. 452-455. IEEE (2016) +4. Blanchette, I., Caparos, S.: When emotions improve reasoning: The possible roles of relevance and utility. Thinking & Reasoning 19(3-4), 399-413 (2013) +5. Bojanowski, P., Grave, E., Joulin, A., Mikolov, T.: Enriching word vectors with subword information. Transactions of the Association for Computational Linguistics 5, 135-146 (2017) +6. Bondarenko, A., Frobe, M., Beloucif, M., Gienapp, L., Ajjour, Y., Panchenko, A., Biemann,¨ C., Stein, B., Wachsmuth, H., Potthast, M., Hagen, M.: Overview of Touche 2020: ´ Argument Retrieval. In: Working Notes Papers of the CLEF 2020 Evaluation Labs (Sep 2020) +7. Braud, C., Denis, P.: Comparing word representations for implicit discourse relation classification. In: Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing. pp. 2201-2211. Association for Computational Linguistics, Lisbon, Portugal (Sep 2015). https://doi.org/10.18653/v1/D15-1262, https://www.aclweb.org/anthology/D15-1262 +8. Gilbert, M.A.: Coalescent argumentation. Routledge (2013) +9. Lawrence, J., Reed, C.: Argument mining: A survey. Computational Linguistics 45(4), 765-818 (2020) +10. Mikolov, T., Chen, K., Corrado, G., Dean, J.: Efficient estimation of word representations in vector space (2013) +11. Mitra, B., Nalisnick, E., Craswell, N., Caruana, R.: A dual embedding space model for document ranking (2016) +12. Ounis, I., Amati, G., Plachouras, V., He, B., Macdonald, C., Johnson, D.: Terrier information retrieval platform. In: Losada, D.E., Fernandez-Luna, J.M. (eds.) Advances in ´ Information Retrieval. pp. 517-519. Springer Berlin Heidelberg, Berlin, Heidelberg (2005) +13. Potthast, M., Gienapp, L., Euchner, F., Heilenkotter, N., Weidmann, N., Wachsmuth, H., ¨ Stein, B., Hagen, M.: Argument search: Assessing argument relevance. In: Proceedings of the 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval. p. 1117-1120. SIGIR'19, Association for Computing Machinery, New York, NY, USA (2019). https://doi.org/10.1145/3331184.3331327, https://doi.org/10.1145/3331184.3331327 +14. Potthast, M., Gollub, T., Wiegmann, M., Stein, B.: TIRA Integrated Research Architecture. In: Ferro, N., Peters, C. (eds.) Information Retrieval Evaluation in a Changing World. The Information Retrieval Series, Springer (Sep 2019). https://doi.org/10.1007/978-3-030-22948-1 5 +15. Villata, S., Cabrio, E., Jraidi, I., Benlamine, S., Chaouachi, M., Frasson, C., Gandon, F.: Emotions and personality traits in argumentation: an empirical evaluation 1. Argument & Computation 8(1), 61-87 (2017) +16. Walton, D.: The place of emotion in argument. Penn State Press (2010) \ No newline at end of file diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/tables/page005-table01.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/tables/page005-table01.png new file mode 100644 index 0000000..8ddda7d Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/tables/page005-table01.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/tables/page005-table02.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/tables/page005-table02.png new file mode 100644 index 0000000..891a9c3 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/tables/page005-table02.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/tables/table-01.txt.md b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/tables/table-01.txt.md new file mode 100644 index 0000000..ce4cef6 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/tables/table-01.txt.md @@ -0,0 +1,6 @@ +| | Argument's share | Stance in | % per share | +|---|---|---|---| +| Sent. | in number (%) | Pro | Con | +| N | 169,152 (57%) | 53% | 47% | +| + | 33,406 (11%) | 58% | 42% | +| - | 94,460 (32%) | 50% | 50% | diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/tables/table-02.txt.md b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/tables/table-02.txt.md new file mode 100644 index 0000000..1049e30 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/191/tables/table-02.txt.md @@ -0,0 +1,6 @@ +| | | | IN-OUT | | | | IN-IN | | +|---|---|---|---|---|---|---|---|---| +| | nDCG | nDCG@5 | nDCG@10 | QrelCov@10 | nDCG | nDCG@5 | nDCG@10 | QrelCov@10 | +| R0 | 0.365 | 0.649 | 0.553 | 6.24 | 0.390 | 0.635 | 0.538 | 6.06 | +| RE | 0.369 | 0.699 | 0.559 | 6.24 | 0.385 | 0.625 | 0.528 | 5.96 | +| RN | 0.337 | 0.517 | 0.456 | 5.18 | 0.359 | 0.500 | 0.437 | 4.94 | diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/figures/img_p4_1.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/figures/img_p4_1.png new file mode 100644 index 0000000..59bec29 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/figures/img_p4_1.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/figures/img_p5_1.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/figures/img_p5_1.png new file mode 100644 index 0000000..468327a Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/figures/img_p5_1.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/figures/img_p6_1.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/figures/img_p6_1.png new file mode 100644 index 0000000..5d65056 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/figures/img_p6_1.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/figures/img_p6_2.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/figures/img_p6_2.png new file mode 100644 index 0000000..5234d25 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/figures/img_p6_2.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/figures/img_p7_1.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/figures/img_p7_1.png new file mode 100644 index 0000000..98f52c1 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/figures/img_p7_1.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/paper.pdf b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/paper.pdf new file mode 100644 index 0000000..1a357e2 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/paper.pdf differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/paper.txt.md b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/paper.txt.md new file mode 100644 index 0000000..9c6633f --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/paper.txt.md @@ -0,0 +1,293 @@ +# University of Amsterdam at CLEF 2020 + +## Notebook for the Touché Lab on Argument Retrieval at CLEF 2020 + +Mahsa S. Shahshahani and Jaap Kamps + +University of Amsterdam {m.shahshahani,kamps}@uva.nl + +### Abstract This paper documents the University of Amsterdam's participation in + +CLEF 2020 Touché Track. This is the first year this track has been introduced at CLEF, and we were attracted to participate in it due to its potentialities for Parliamentary debates we are currently working on. This track consists of two tasks: Conversational Argument Retrieval and Comparative Argument Retrieval. We submitted a run to both tasks. For the first task, we used a combination of the traditional BM25 model and learning to rank models. BM25 model helps to retrieve relevant arguments, and learning to rank model helps to re-rank the list and put stronger arguments on top of the list. For the second task, Comparative Argument Retrieval, we proposed a pipeline to re-rank documents retrieved from Clueweb using three features: PageRank scores, web domains, and argumentativeness. Preliminary results on 5 queries have shown that this heuristic pipeline may help to achieve a balance among three important dimensions: relevance, trustworthiness, and argumentativeness. + +## 1 Introduction + +We believe that we passed the era in which search engines were supposed to only give us a ranked list of documents or answers and they have more potentialities to help us in decision making process. Argument retrieval task has been defined to formulate this problem. Touché track at CLEF [3] offers an opportunity to work on this interesting problem having access to a debate corpus from two different points of view: Looking for different views about a problem in debates between opponents and supporters of a controversial issue, and looking for comparative opinions about different alternative. This track consists of a different task (two tasks in total) for each point of view and we submitted a run to both tasks. + +In this paper, we cover both tasks; first, we give a high-level summary of the first task, followed by our detailed approach. Then, we cover the same for the second task. Finally, we will conclude the paper by mentioning our main contributions and findings. + +*We will add results section whenever the results would be out.* + +Copyright c 2020 for this paper by its authors. Use permitted under Creative Commons License Attribution 4.0 International (CC BY 4.0). CLEF 2020, 22-25 September 2020, Thessaloniki, Greece. + +----- + +## 2 Conversational Argument Retrieval + +For detailed information about CLEF track's experimental setup, we refer to the overview paper [4] and to the track homepage.1 However, we provide a high-level summary to make this paper self-contained. + +### 2.1 Task definition + +The goal of this task is to retrieve relevant arguments from online debate portals, given a query on a controversial topic. *Corpus Args.me [1] has been created by crawling arguments from 7 debate websites.* It includes 387,606 arguments taken from 59,637 debates. A search engine based on Elasticsearch has been set to make it easier to work with this corpus [10]. This search engine ranks arguments using BM25 ranking algorithm. Different approaches can be used later to re-rank these retrieved arguments. *Queries 50 controversial topics have been picked for this task. Each topic has both pro* and con arguments in the corpus. *Quality Assessment Proposed approaches are supposed to retrieve "strong" arguments.* An argument is consiagstuhl ered strong if it is topically relevant, logically cogent, rhetorically well-written, and useful to help in stance-building process. Here, we define these assessment dimensions taken from [9] and the annotation guidelines for Dagstuhl- 15512 ArgQuality Corpus.2 *Topical relevance: As every other ranking task, retrieved arguments should provide the* user with relevant information about the query. + +Besides relevance, in general, there are three main dimensions for assessing the quality of arguments: logic, rhetoric, and dialectic. *Logical cogency: An argument with acceptable premises that are relevant and sufficient* to the argument's conclusion is considered "cogent" [7]. *Rhetorical well-writtenness: An argument is called "rhetorically well-written" if it is* effective and successful in persuading a target audience of a conclusion [2]. *Dialectic: An argument is considered reasonable if it contributes in the users' stance-* building process regards a given issue in a way that is acceptable to everyone. + +### 2.2 Our Approach + +We treated this task as a re-ranking problem. In the absence of training data, we have to use unsupervised approaches. However, we used an existing debate dataset created for studying argument quality assessment to train a classifier. First, we describe this corpus. Later, we explain our approach in three consecutive steps. + +1 https://events.webis.de/touche-20/shared-task-1.html 2 http://www.arguana.com + +----- + +*Corpus Dagstuhl-15512 ArgQuality [9] includes 20 arguments for each of 16 queries.* Three annotators have annotated these 320 arguments on 15 dimensions with three labels. However, we only use four dimensions: cogency, effectiveness, reasonableness, and overall quality. The set of labels includes ordinal scores from 1 (low) to 3 (high) for all dimensions. We used majority voting technique to get the label for each dimension, and substituted the ones labeled as 3 with 2 as there are a very few samples with label "3" in the corpus . + +**Approach We created the final ranked list in three steps. Before explaining each step,** + +we explain the way we represented arguments. + +***Argument Representation: We used pre-trained BERT-base [6] model from Hugging-*** + +Face Transformers framework in python to represent arguments. As BERT model imposes a limit on the length of documents after tokenization, we used the first 512 tokens in an argument if its length exceeds this limit. + +***First step- Ranking: BM25 is the traditional unsupervised ranking model which scores*** + +the relevancy of documents (here arguments) in regards to queries based on the frequency of common terms between the query and argument. We ranked arguments for each topic based on BM25 using args.me search engine. + +***Second step- Classification: We trained a classifier on Dagstuhl-15512 ArgQuality*** + +corpus to recognize and label cogency, well-writtenness, reasonableness and overall quality of each argument. Later, we applied this classifier to all retrieved arguments in the ranked list from the fist step. + +We got the majority voting for each dimension; in all of the arguments the majority for all four dimensions are the same. It is aligned with the conclusion in the main paper [9] which indicates that cogency, effectiveness, and reasonableness correlate strongly with overall quality, and also much with each other. + +We trained two classifiers on 90% of data: A decision tree classifier, and an SVM classifier. The accuracy on the remaining 10% of data has been shown in Table 1. We used scikit-learn framework for python 3 to train both classifiers. For decision tree, we used "gini" criterion, and set minimum required samples to split to 2. For SVM, we used "rbf" kernel, and set regularization parameter to 1. We selected SVM classifier for the further step. + +***Third step-Re-ranking: In the third step, we re-ranked retrieved arguments from the*** + +first step using learning-to-rank models. We use the output of step 2 as a feature in step 3 to re-rank the arguments. We trained three different learning to rank models: Ranknet, RandomForests, and LambdaRank. We used RankLib3 library with default sets of parameters to apply these models. In order to train learning to rank models, we used argument representations based on BERT model, the output of the second step, and two additional features based on named entities. We defined two binary features indicating the presence of numerical named entities (percent, quantity, money) and other + +3 https://github.com/codelibs/ranklib + +----- + +entities (person, location, organization) in the argument. We showed the number of arguments with and without these entities in Figure 1 and Figure 2. These figures show the difference in the distribution of each of these features in strong (label=2) and weak (label=1) arguments. Arguments using these kinds of entities are more likely providing users with persuasive and effective information to make their stance, and this can lead to a more probability to be labeled as "strong". + +We trained learning to rank models on Dagstuhl dataset, and applied the best model (Ranknet) to re-rank retrieved arguments from args.me dataset for all 50 topics in the shared task. We trained models on 90% of data and reported accuracy on the remaining 10% of data in Table 2. + +**Table 1. classifiers** + +Model Accuracy + +Decision Tree 0.43 SVM 0.53 + +![](../figures/paper_207/img_p4_1.png) + +**Figure 1. Having/lacking numerical named entities (percent, money, quantity)** + +Model + +### RankNet + +Random Forests LambdaRank + +**Table 2. LTR** + +NDCG@1 NDCG@5 + +**0.873 0.9587** + +0.7333 0.9155 0.7778 0.9291 + +----- + +![](../figures/paper_207/img_p5_1.png) + +**Figure 2. Having/lacking other named entities (person, organization)** + +### 2.3 Results + +In the final relevance judgments, 30 documents for each topic have been annotated using 6 labels: -2,1,2,3,4 and 5. + +**Classifier We ran our classifier that was trained on Dagstuhl-15512 ArgQuality corpus** + +on arguments from judgments to see if the results are correlated with the final judgment. Unfortunately, we observed that the classifier does not work well and return label '1' for more than 90% of the judged arguments. This suggests that the classifier does not play a role in the final results of our learning to rank model. This is not surprising as the dataset we trained our SVM classifier on is very different from the test set. The arguments in the training set are very short and consist of only one to three sentences, while we are labeling a set of complete documents in the test set. However, as the BERT representation has a limit on the size of the input text, for the longer texts we only used the first 512 tokens of each argument. Using more advanced approaches such as averaging over sliding windows of 512 tokens might make the classifier useful. + +**Entities Similar to Figure 1 and Figure 2, we looked into the distribution of numeri-** + +cal and other types of entities in relevant and non-relevant documents. We considered documents with label '-2' as non-relevant, and other documents in the judgment file as relevant. Results have been shown in Figure 3 and 4. As it is obvious from the figures, the distribution of entities is the opposite of what we observed in the dataset we used at the time of developing our model. However, it still can be used as a feature as it shows a little difference between relevant and non-relevant documents. + +### Query Length We looked into per query results to get an insight into the way our + +model works. We used NDCG@5 metric as it has been the main metric for the shared task. + +----- + +![](../figures/paper_207/img_p6_1.png) + +**Figure 3. Having/lacking numerical named entities (percent, money, quantity)** + +![](../figures/paper_207/img_p6_2.png) + +**Figure 4. Having/lacking numerical named entities (person, organization)** + +**Table 3. Results-NDCG@10 metric** + +--- + +### Model NDCG@1 NDCG@5 NDCG@10 MAP + +--- + +UvATask1LTR 0.5214 0.5548 0.3709 0.1129 + +--- + +As we used the whole topic (without removing stop words), the model works better for shorter queries (Figure 5). + +----- + +![](../figures/paper_207/img_p7_1.png) + +**Figure 5. Average Query Length for the different ranges of NDCG@5)** + +## 3 Comparative Argument Retrieval + +Similar to the previous section, for detailed information about this task's experimental setup, we refer to the overview paper [4] and to the track homepage.4 However, we provide a high-level summary. + +### 3.1 Task definition + +The goal of this task is to retrieve and rank documents from web that help to answer a comparative question from "everyday life". + +*Corpus Clueweb12 is a dataset created by crawling 733,019,372 web documents seeded* with 2,820,500 urls from Clueweb09 [5]. We used a publicly available search engine [8] based on Elasticsearch to retrieve documents from Clueweb12 based on BM25 ranking model. + +*Queries 50 comparative topics from everyday life have been picked for this task.* + +### 3.2 Our approach + +We treated this task as a re-ranking problem. In the absence of training data, we have to use unsupervised approaches. We labeled retrieved documents for 5 topics to have an insight of how our heuristic approach works. + +In the first step, we used ChatNoir search engine to rank documents retrieved for each topic. In the second step, we used three different features to re-rank them. Here, we introduce the features we used, followed by our heuristic approach to combine them and create the final ranked list. + +4 https://events.webis.de/touche-20/shared-task-2.html + +----- + +### Argumentativeness We trained a simple SVM classifier based on data from args.me + +corpus and Clueweb to distinguish between argumentative and non-argumentative documents. Similar to the first task, we used BERT-based model from HuggingFace Transformers library for Python to represent documents, and we used the first 512 tokens in a document if its length exceeds the limit of BERT model. + +To train the classifier, we used a small sample from each corpus. These samples are created by submitting all 50 controversial queries in the first task to both corpora and got up to 100 documents for each query. Then, we manually removed argumentative documents from the sample taken from Clueweb and considered the remaining documents as negative examples. All retrieved documents from args.me corpus have been considered as positive examples. The final training set consists of 3000 positive and 3000 negative examples. Then, We trained a simple SVM classifier on 80% of the data, and evaluated it on the remaining 20% of documents. It achieved 87% in terms of accuracy. All parameters for the argumentativeness classifier have been set to their default values in Scikit-learn5 library for Python. + +### Web domains Clueweb has been formed by crawling web documents with some post- + +filters. But, the goal of this task is to retrieve documents including personal opinions or suggestions. Thus, documents from particular domains like Wikipedia are not desirable. On the contrary, documents from discussion forums, debate websites, and blogs can be very helpful. Having this intuition in mind, we defined a binary feature that indicates if the source URL for a discussion contains 'forum' or 'blog' terms to give a bonus to web pages from discussion forums or blogs. + +***PageRank Although desired documents are those from discussion forums and personal*** + +blogs, they should also be trustworthy. To take trustworthiness into account, we used page rank scores to prioritize documents taken from more reliable sources. In ChatNoir search engine, every returned document has been associated with a PageRank score. We directly used these returned scores. + +***Re-ranking We introduced three features, and our goal is to re-rank documents based*** + +on a combination of these features (argumentativeness, domain addresses , and PageRank scores). To generate the final ranked list, we make a heuristic ranking pipeline in four steps: + +**- The 1st step: The initial ranked list is taken from ChatNoir search engine. ChatNoir** + +retrieves documents from Clueweb and ranks them using traditional BM25 ranking model. We use the whole topic title as the query. We also examined submitting the query after removing stop words or just using the entities in the topic title. But, using the whole topic title worked better. + +**- The 2nd step: Page-rank scores are used to re-rank the list from the first step in** + +descending order. This may result in putting a document, initially ranked very low, on top of the list. In order to avoid this, moving documents in the ranked list is limited to a maximum of 10 positions. + +5 https://scikit-learn.org/stable/supervised\_learning.html#supervised-learning + +----- + +**Table 4. Preliminary Results- NDCG@10 metric** + +| Model | Relevance | Argumentativeness | Trustworthiness | +|---|---|---|---| +| initial | 0.87 | 0.71 | 0.81 | +| pagerank | 0.89 | 0.66 | 0.87 | +| domain | 0.84 | 0.72 | 0.80 | +| argumentative classifier | 0.80 | 0.84 | 0.79 | +| mixed | 0.84 | 0.78 | 0.82 | + +--- + +**- The 3rd step: Web domain are used to re-rank ranked list from the second step. To** + +do this, the documents with positive domain feature (which means the document is taken from a blog or discussion forum) are put on top of the list. This has been performed within every 10 documents in the list. We split the list into chunks of 10 documents and we keep their relative positions. + +**- The 4th step:All retrieved documents in the ranked list from the third step are clas-** + +sified using the argumentativeness classifier we have trained. Documents classified as positive are put on top of the list. We keep their relative positions. This has been performed within every chunk of 10 documents in the list. + +We put this limits on moving documents in the list with this intuition in mind that relevance should be prioritized in comparison with trustworthiness and argumentativeness. + +### 3.3 Results + +Since the final judgment file does not consist of separate judgment lists for three different dimensions (relevance, trustworthiness, and argumentativeness), we include our preliminary results too. + +**Preliminary Results: To gain an insight into the effectiveness of our heuristic model,** + +we manually labeled 10 retrieved documents for 5 queries. We labeled documents using three labels: 0 for non-relevant, non-argumentative, or untrustworthy; 1 for relevant, argumentative, or trustworthy, and 2 for highly-relevant, highly-argumentative, or highly trustworthy. + +We evaluated the top 10 documents for each ranked list: BM25, re-ranked by PageRank scores, re-ranked by web domains, re-ranked by argumentativeness, and mixed model. + +Evaluation results have been reported in Table 3. The heuristic mixed model does not achieve the same performance as BM25 in terms of relevance, the same performance as the argumentative classifier model in terms of argumentativeness, and the same performance as the PageRank model in terms of trustworthiness. But, it seems that it struck a balance among all three dimensions. + +----- + +**Table 5. Final Results** + +--- + +| Model | NDCG@1 | NDCG@5 | NDCG@10 | +|---|---|---|---| +| initial | 0.5068 | 0.4480 | 0.4196 | +| pagerank | 0.4723 | 0.4256 | 0.4087 | +| domain | 0.4100 | 0.3845 | 0.4132 | +| argumentative classifier | 0.4281 | 0.4123 | 0.4023 | +| UvATask2SVM | 0.5000 | 0.4464 | 0.4185 | + +--- + +**Final Results: The final results for every step of our model have been reported in** + +Figure 5. Since three different aspects had been defined for evaluation of this task, We expected to receive three judgment sets. However, the judgment file issued only the overall relevance, which we sacrificed for the other two dimensions. + +Initially, we processed the data and implemented our models based on the three different aspects which had been defined previously. Those include relevance, trustworthiness, and argumentativeness. Given the fact that we received the overall relevance file only, we extracted our results purely based on the relevance factor for the evaluation of this task. This categorically implies that the obtained results could have been more optimized if we had performed our preliminary assessment based on one aspect only. + +## 4 Conclusion + +This paper documents our first participation in the Touché 2020 Track. We explained our approaches for both tasks in the track: Conversational Argument Retrieval, and Comparative Argument Retrieval. + +For the Conversational Argument Retrieval task, we used an existing argument quality assessment dataset to train a classifier and re-rank arguments based on the output of this classifier. We showed that named entities are important features to distinguish between strong and weak arguments on the preliminary data. But, the final results show that neither classifier nor entities do not help. However, it is worth mentioning that the classifier had been trained on a completely different set of arguments. So, if we train it on the same data from args.me, it might be useful for the final results. This needs further investigations to be proved. + +For the Comparative Argument Retrieval task, we introduced three features to rerank arguments taken from Clueweb. We proposed a pipeline to combine different aspects (relevance, trustworthiness, and argumentativeness) to create the final ranked list. Preliminary results have shown that this heuristic pipeline may successfully strike a balance between all three dimensions. However, the final evaluation has been done only on relevance. This caused our method to be sub-optimal. + +We hope and expect that the valuable bench-marking data created at Touché track will be of great value to motivate, and greatly facilitate, further research into argument retrieval. + +----- + +### Acknowledgments + +*This research was supported in part by the Netherlands Organization for Scientific Research (NWO, grant # CISC.CC.016, ACCESS project). Views expressed in this paper are not necessarily shared or endorsed by those funding the research.* + +## References + +1. Ajjour, Y., Wachsmuth, H., Kiesel, J., Potthast, M., Hagen, M., Stein, B.: Data acquisition for argument search: The args.me corpus. In: Benzmüller, C., Stuckenschmidt, H. (eds.) KI 2019: Advances in Artificial Intelligence - 42nd German Conference on AI, Kassel, Germany, September 23-26, 2019, Proceedings. Lecture Notes in Computer Science, vol. 11793, pp. 48-59. Springer (2019). https://doi.org/10.1007/978-3-030-30179-8\_4, https://doi.org/10.1007/978-3-030-30179-8\_4 +2. Blair, J.A.: Groundwork in the theory of argumentation: Selected papers of J. Anthony Blair, vol. 21. Springer Science & Business Media (2012) +3. Bondarenko, A., Fröbe, M., Beloucif, M., Gienapp, L., Ajjour, Y., Panchenko, A., Biemann, C., Stein, B., Wachsmuth, H., Potthast, M., Hagen, M.: Overview of Touché 2020: Argument Retrieval. In: Working Notes Papers of the CLEF 2020 Evaluation Labs (Sep 2020) +4. Bondarenko, A., Hagen, M., Potthast, M., Wachsmuth, H., Beloucif, M., Biemann, C., Panchenko, A., Stein, B.: Touché: First shared task on argument retrieval. In: Jose, J.M., Yilmaz, E., Magalhães, J., Castells, P., Ferro, N., Silva, M.J., Martins, F. (eds.) Advances in Information Retrieval - 42nd European Conference on IR Research, ECIR 2020, Lisbon, Portugal, April 14-17, 2020, Proceedings, Part II. Lecture Notes in Computer Science, vol. 12036, pp. 517-523. Springer (2020). https://doi.org/10.1007/978-3-030-45442-5\_67, https://doi.org/10.1007/978-3-030-45442-5\_67 +5. Clarke, C.L.A., Craswell, N., Soboroff, I.: Overview of the TREC 2009 web track. In: Voorhees, E.M., Buckland, L.P. (eds.) Proceedings of The Eighteenth Text REtrieval Conference, TREC 2009, Gaithersburg, Maryland, USA, November 17-20, 2009. NIST Special Publication, vol. 500-278. National Institute of Standards and Technology (NIST) (2009), http://trec.nist.gov/pubs/trec18/papers/WEB09.OVERVIEW.pdf +6. Devlin, J., Chang, M., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. In: Burstein, J., Doran, C., Solorio, T. (eds.) Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2019, Minneapolis, MN, USA, June 2-7, 2019, Volume 1 (Long and Short Papers). pp. 4171-4186. Association for Computational Linguistics (2019). https://doi.org/10.18653/v1/n19-1423, https://doi.org/10.18653/v1/n19-1423 +7. Johnson, R.H., Blair, J.A.: Logical self-defense. International Debate Education Association (2006) +8. Potthast, M., Hagen, M., Stein, B., Graßegger, J., Michel, M., Tippmann, M., Welsch, C.: Chatnoir: a search engine for the clueweb09 corpus. In: Hersh, W.R., Callan, J., Maarek, Y., Sanderson, M. (eds.) The 35th International ACM SIGIR conference on research and development in Information Retrieval, SIGIR '12, Portland, OR, USA, August 12-16, 2012. p. 1004. ACM (2012). https://doi.org/10.1145/2348283.2348429, https://doi.org/10.1145/2348283.2348429 +9. Wachsmuth, H., Naderi, N., Hou, Y., Bilu, Y., Prabhakaran, V., Thijm, T.A., Hirst, G., Stein, B.: Computational argumentation quality assessment in natural language. In: Proceedings + +----- + +of the 15th Conference of the European Chapter of the Association for Computational Linguistics, EACL 2017, Valencia, Spain, April 3-7, 2017, Volume 1: Long Papers. pp. 176-187 (2017). https://doi.org/10.18653/v1/e17-1017, https://doi.org/10.18653/v1/e17-1017 + +10. Wachsmuth, H., Potthast, M., Khatib, K.A., Ajjour, Y., Puschmann, J., Qu, J., Dorsch, J., Morari, V., Bevendorff, J., Stein, B.: Building an argument search engine for the web. In: Habernal, I., Gurevych, I., Ashley, K.D., Cardie, C., Green, N., Litman, D.J., Petasis, G., Reed, C., Slonim, N., Walker, V.R. (eds.) Proceedings of the 4th Workshop on Argument Mining, ArgMining@EMNLP 2017, Copenhagen, Denmark, September 8, 2017. pp. 49-59. Association for Computational Linguistics (2017). https://doi.org/10.18653/v1/w17-5106, https://doi.org/10.18653/v1/w17-5106 \ No newline at end of file diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/tables/page006-table01.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/tables/page006-table01.png new file mode 100644 index 0000000..6239a2e Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/tables/page006-table01.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/tables/page009-table01.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/tables/page009-table01.png new file mode 100644 index 0000000..f6ed984 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/tables/page009-table01.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/tables/page010-table01.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/tables/page010-table01.png new file mode 100644 index 0000000..54ec9db Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/tables/page010-table01.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/tables/table-01.txt.md b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/tables/table-01.txt.md new file mode 100644 index 0000000..7b032fb --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/tables/table-01.txt.md @@ -0,0 +1,7 @@ +| Model | Relevance | Argumentativeness | Trustworthiness | +|---|---|---|---| +| initial | 0.87 | 0.71 | 0.81 | +| pagerank | 0.89 | 0.66 | 0.87 | +| domain | 0.84 | 0.72 | 0.80 | +| argumentative classifier | 0.80 | 0.84 | 0.79 | +| mixed | 0.84 | 0.78 | 0.82 | diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/tables/table-02.txt.md b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/tables/table-02.txt.md new file mode 100644 index 0000000..64cb26d --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/207/tables/table-02.txt.md @@ -0,0 +1,7 @@ +| Model | NDCG@1 | NDCG@5 | NDCG@10 | +|---|---|---|---| +| initial | 0.5068 | 0.4480 | 0.4196 | +| pagerank | 0.4723 | 0.4256 | 0.4087 | +| domain | 0.4100 | 0.3845 | 0.4132 | +| argumentative classifier | 0.4281 | 0.4123 | 0.4023 | +| UvATask2SVM | 0.5000 | 0.4464 | 0.4185 | diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/paper.pdf b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/paper.pdf new file mode 100644 index 0000000..3686150 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/paper.pdf differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/paper.txt.md b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/paper.txt.md new file mode 100644 index 0000000..33c96fb --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/paper.txt.md @@ -0,0 +1,333 @@ +# Exploring Argument Retrieval with Transformers + +## Notebook for the Touché Lab on Argument Retrieval at CLEF 2020 + +Christopher Akiki and Martin Potthast + +Leipzig University + +### Abstract We report on our recent efforts to employ transformer-based models + +as part of an information retrieval pipeline, using argument retrieval as a benchmark. Transformer models, both causal and bidirectional, are independently used to expand queries using generative approaches as well as to densely embed and retrieve arguments. In particular, we investigate three approaches: (1) query expansion using GPT-2, (2) query expansion using BERT, and orthogonal to these approaches, (3) embedding of documents using Google's BERT-like universal sentence encoder (USE) combined with a subsequent retrieval step based on a nearest-neighbor search in the embedding space. A comparative evaluation of our approaches at the Touché lab on argument retrieval places our query expansion based on GPT-2 first on the leaderboard with a retrieval performance of 0.808 nDCG@5, improving over the task baseline by 6.878%. + +## 1 Introduction + +Search has become the sine qua non tool of information access, and the gateway to the World Wide Web. The users of web search engines meanwhile expect a high quality of the search results in terms of their relevance to the queries submitted: If relevant documents exist for a given query, they are usually found, and the most relevant ones can be expected to be ranked highest. However, search engines are optimized for ad hoc retrieval tasks, and a key assumption is that a single document suffices to satisfy the information need underlying a query. That assumption falls apart when the topic of interest is inherently subjective and nuanced, such as is the case for contentious issues. Any one document will most likely argue one way or another, so that a user may need to peruse many documents at a time to satisfy a deliberative information need. The current search landscape is, however, not especially attuned to such a nuance, usually preferring to let "the stakeholders compete for what opinion ranks higher" [23]. The ability to specifically handle arguments rather than the documents that might contain them is an attempt to address that problem using computational argumentation analysis [23, 31]. Such an approach forms the basis of the args.me search engine which relies on a corpus of more than 300,000 arguments mined from online debate portals [1]. This corpus formed the basis of the Touché shared task on argument retrieval [3]. + +Copyright c 2020 for this paper by its authors. Use permitted under Creative Commons License Attribution 4.0 International (CC BY 4.0). CLEF 2020, 22-25 September, Thessaloniki, Greece. + +----- + +We leverage transformer models to see how they might enrich argument retrieval at various stages of the information retrieval (IR) pipeline. Attention-based transformer models [30] have seen a recent surge in popularity as their readiness for massive parallelization and the increasing availability of computational power on GPUs led to such models claiming the state-of-the-art crown on many natural language processing and understanding tasks [7, 25]. We show that competitive performance can be achieved on this task with no preprocessing of the documents or fine-tuning of the models on the task. In what follows, Section 2 presents related work, Section 3 exposes our approach, and Section 4 presents the results of our participation in the shared task. + +## 2 Related Work + +Although the subject of arguments in a retrieval context is by no means a new development (see, e.g., [26] and [28]), the field itself is still nascent. Lawrence and Reed [18], Potthast et al. [23], and Wachsmuth et al. [31] provide a comprehensive overview of the field, with the last two further putting forth the theoretical considerations that underlie the development of a tool where arguments form the unit of retrieval, using the args.me dataset [1] to ground the theory in an applied retrieval study. Another salient research direction is spearheaded by IBM's Project Debater and corresponding studies that leverage its numerous datasets (see [29] for an overview of the project). + +The growing interest of natural language processing (NLP) researchers in information retrieval and argument mining [27], as well as the impressive performance and ease of use of transformer-based models [32] and their propensity to semantic nuance makes a convergence of the two unavoidable. Indeed, researchers in IBM's project debater have recently published two corresponding papers [9, 14] that both use BERT [7] for argument retrieval and argument quality assessment. Like other efforts in the argument mining direction [6, 11], these methods, though impressive in their results, do not readily transfer to a domain other than that of the retrieval of arguments because they are usually fine-tuned on a specific document corpus. + +The use of transformers in general, specifically BERT, to the field of document retrieval was until very recently limited to frameworks where initial retrieval is delegated to the Lucene-based Anserini retrieval toolkit [34, 33], which, while proving a promising approach, did not attempt to instrumentalize transformers at different parts of the IR pipeline. Another approach similarly leverages BERT and Anserini for ad hoc document retrieval, while also coupling the approach with the ability to interface with Python to simplify neural network based research [35]. + +The semantic prowess of transformers makes them prime candidates for enriching IR pipelines that rely on query expansion [2]. For a deeper coverage of query expansion we refer the reader to Azad and Deepak's [2] thorough survey on the topic. In brief, query expansion consists in augmenting a user query to increase its effectiveness and reduce ambiguity. That is achieved through reformulating the query using additional data, and the source of that data coincides with the different sub-approaches of this sub-field of IR. Azad and Deepak [2] differentiate between two main directions: global analysis and local analysis. The latter relies on user feedback, both direct and implicit, whereas the former consists of approaches that rely on knowledge that can be gleaned + +----- + +and derived from the query itself or its context. These include linguistic, corpus-based, log-based and web-based approaches. Global analysis has proven to be of particular interest for transformer-focused research [8, 22, 19, 21]. The work of Dibia [8] in particular aligns nicely with our approach in Section 3.2. Our approaches differ however in the strategies used to determine where and how to inject context. To our knowledge, the query expansion approach we develop in Section 3.1 is the first use of a transformer decoder (GPT-2 in this instance) to generate documents that read as though they might have originated from a corpus of interest, at least plausibly enough for a retrieval system, and thus narrow down the scope of search. + +## 3 Using Transformers for Document Retrieval + +We set out to instrumentalize for the retrieval of documents the ability of different transformers to encode knowledge. The original transformer introduced by Vaswani et al. [30] uses a standard seq2seq encoder-decoder architecture, whereby the encoder learns a task-specific embedding, and the decoder learns a language model. Subsequent transformer-based models do not necessarily follow that convention. BERT [7] only uses an encoder, and GPT-2 [25] only a decoder. It is therefore useful to qualify the rather general "transformer" nomenclature with either "encoder" or "decoder". + +We use transformer decoders (i.e., GPT-2-like models) for query expansion via text hallucination (Section 3.1), i.e., the generation of a text that reads as if it might have come from the corpus, and transformer encoders (i.e., BERT-liked models) for keyword-based query expansion (Section 3.2). Moreover, we consider transformer encoders for document embedding (Section 3.3). Both query expansion approaches make use of an Elasticsearch index where the documents of the args.me corpus were indexed using a language model with Dirichlet priors (DirichletLM) [36], which has been shown to be superior to other retrieval models for retrieving arguments from the args.me corpus [23]. The embedding approach, on the other hand, uses a vector-based similarity index for retrieval. + +Self-supervised pre-training on massive amounts of qualitatively diverse data is what enables transformer models to encode the knowledge that allows them to perform as well as they do on natural language processing (NLP) and natural language understanding (NLU) tasks. Therein also lies their promise for retrieval tasks. We make the conscious decision to only rely on the knowledge encoded into the models by these tasks; that is, we do not rely on any argumentation-specific fine-tuning. That allows us to gauge the performance of transformers for retrieval tasks in general. In particular, our investigation aims at a modular proof-of-concept approach, not to show the superiority of a certain method over others, nor was it our goal to optimize the ensuing pipeline for accuracy on the relevance judgments. + +### 3.1 Query Expansion with a Transformer Decoder + +A lot of the recent media hype around transformer networks is centered around their ability to generate coherent text. Transformer decoders can be trained as causal language models (CLM), seeing as the representation of a given token can only depend + +----- + +**Table 1. Language model prompts for argumentative query expansion. The dots (. . . ) indicate** + +where the language model takes over and generates text that is used as expanded query. + +--- + +### Positive Prompt + +--- + +- What do you think? - Yes because . . . + +--- + +- What do you think? - The answer is yes . . . + +### Negative Prompt + +- What do you think? - No because . . . - What do you think? - The answer is no . . . + +### Neutral Prompt + +- What do you think? - I don't know . . . - What do you think? - Not sure . . . + +--- + +on the tokens preceding it. Indeed, since transformer decoders like GPT-2 simulate language models, it is possible to iteratively generate sentences by sampling from the output distribution at a given time step and feeding that output back into the network at the following time step. By choosing an opening for a text (called prompt), it becomes thus possible to steer GPT-2 and exert some influence over the text it generates. Our approach for query expansion makes use of this capability to generate argumentative expansions of a given query with a positive, negative, and a neutral stance by prompting GPT-2 in turn with the six prompts shown in Table 1. The prompts are purposefully constructed so as to simulate an argumentative dialog that is to be completed by GPT-2. The quality of generated sequences, and thus also of our retrieval results, is highly contingent upon the way in which tokens are sampled from the network. It is easy for neural language models to degenerate into incoherent, non-human sounding text when using naive likelihood maximization [16]. Many sampling methods exist that try to make sure the generated text is less likely to be incoherent, repetitive, or overly generic [12, 13, 16]; the following sampling approaches are the most salient ones, ordered from basic to advanced: + +### - Pure sampling is the most naive as well as often the worst-performing sampling + +method. It consists of greedily choosing the most likely token at every time step. This sampling method usually leads to low quality text. + +**- Beam Search is a heuristic that keeps a set number of candidate sequences until** + +they all reach the desired length and keeps the most likely candidate. Search for the output sequence that actually maximizes likelihood is intractable, so that beam search provides a reasonable alternative in practice. + +**- Temperature scaling reshapes the output distribution by skewing it either toward** + +high probability tokens or low probability tokens: the former improves generation quality but hurts diversity [16], while the latter regularizes generation by making the model less certain of its top choices. + +**- Top-k sampling is the sampling method that was used for GPT-2 [25]. It truncates** + +the language model's output distribution to a set of size k of most likely tokens. The sampling then uses relative probability within this truncated set. All other tokens are not considered for generation. + +**- Nucleus sampling is a stochastic decoding method proposed by Holtzman et al.** + +[16]. Similar to top-k sampling, it also truncates the output distribution, albeit by setting a threshold the cumulative distribution of top tokens must reach before discarding the rest of the tokens. + +----- + +For each one of the six prompts in Table 1, we use GPT-2 to generate four possible continuations up to a maximum of 100 tokens. To ensure the four continuations are different from each other, we chose different combinations of the aforementioned sampling strategies.1 Having framed the prompts as conversational (by virtue of the dashes and formatting), the ensuing generated text often tends to read like an argument. Having generated 24 such texts,2 we discard the original query and use these hallucinations as 24 queries against the DirichletLM index, which we combine additively to generate the final rankings. We only consider those documents which were returned by at least twelve of the queries simultaneously.3 For illustration purposes, consider the query "Can alternative energy effectively *replace fossil fuels?". Three of the 24 texts that our approach generates are:* + +**- Yes, because it has proven to be a significant and lasting improvement in fuel effi-** + +*ciency, carbon neutrality. The only other thing that could possibly help this energy is the need for nuclear reactors at low cost which would require more than 20 percent of current generation electricity by 2030 (currently under construction), plus less renewable resources like wind or solar power as well but with sufficient amounts of coal/solar panels if there are enough [sic]* + +**- No, because there is no evidence for that. There are a few possible alternative** + +*energy options available to people who would like to cut down fossil fuels, and I believe those include wind or solar power."The main thing we want in our future climate policies has to be better use of resources instead on these things than if they weren't used at the moment," said Kieferlein. "We need clean air [sic]* + +**- Not sure. However, many scientists have made the point that alternative sources** + +*of power are already producing more carbon emissions than they would otherwise (and it seems like such a small number in our country). There has been some debate about whether this was actually true or if there simply wasn't much coal available at all to replace fossil fuels and other forms thereof as an environmentally sustainable form. . . In fact, recent studies suggest we [sic]* + +### 3.2 Query Expansion with a Transformer Encoder + +Unlike transformer decoders, transformer encoders like BERT [7] cannot be used to auto-regressively generate sentences. Their attention-based architecture is such that any token can see ("attend to") any other token of the sentence. Being able to look ahead into the future of a sequence breaks the causality required by a CLM. As such, these models have to rely on other pre-training tasks to gain linguistic coherence. Global linguistic coherence is achieved through next sentence prediction, where BERT has to predict whether two sentences follow each other in a corpus. Local coherence is achieved through another pre-training task, namely masked language modeling (MLM). + +1One with greedy sampling using 10 beams, and three using a temperature of 1.6, a top-k threshhold of 100 tokens, and a nucleus sampling probability threshhold of 0.4. 2 Six prompts with four continuations per prompt. + +3This corresponds to an Elasticsearch boolean query of type "should" with the min\_should\_match + +parameter set to 12. + +----- + +**Table 2. Masked language model prompts for argumentative query expansion. The masks** + +([MASK]) indicate where the language model takes over and generates alternative words that are used as expanded query. + +--- + +### Stance Prompt + +--- + +- What do you think? - Yes, because of [MASK] and the benefits of [MASK]. + +--- + +- What do you think? Positive + +- Absolutely, I think [MASK] is good! + +--- + +- What do you think? - Yes, [MASK] is associated with [MASK] during [MASK]. + +--- + +- What do you think? - No, because of [MASK] and the risk of [MASK] [MASK]. + +--- + +- What do you think? Negative + +- Absolutely not, I think [MASK] is bad! + +--- + +- What do you think? - No, [MASK] is associated with [MASK] during [MASK]. + +--- + +- What do you think? - What about [MASK] or [MASK]? Neutral + +--- + +- What do you think? - Don't forget about [MASK]! + +--- + +During training, tokens in the input are masked at random and the network is tasked with guessing what that word was. Learning to fill in the blank has been shown to imporve + +the quality of text generation [10]. We leverage this ability of the model to "fill in the blank" to enrich the original + +query with a set of words that are contextually relevant to the topic at hand. To achieve that, we again augment the original query using the same strategy as outlined in the previous section, this time, however, we leave blanks for BERT to fill out in the form of the [MASK] token (see Table 2). For every [MASK] in every augmented seed text, we ask BERT to return the five most likely words, filtering out stop words, punctuation, and sub-words. This amounts to an average (min=206, max=473) of 340 thematic keywords per query. All keywords are then joined together into a space-separated list of keywords which is what we use to query the DirichletLM index, discarding the original query. For illustration, consider the query "Can Alternative Energy Effectively Replace Fossil + +*Fuels?" and provide the resulting keywords when expanding the query with BERT:* + +*diesel, cost, nuclear, consumption, hydrogen, technologies, energy, future, electricity, pregnancy, coal, alternative, migration, emissions, efficiency, economics, technology, growth, wartime, earthquakes, green, environmental, accidents, costs, renewable, winter, development, pollution, new, stress, water, oil, accident, death, health, warming, sustainability, accidental, fires, competition* + +### 3.3 Document Embedding with a Transformer Encoder + +Our final approach also employs a transformer encoder in the form of the large variant of Google's universal sentence encoder (USE) [5]. Though architecturally similar to + +----- + +BERT, the USE model is trained with very different pre-training tasks, which were specifically picked to perform well on downstream NLU tasks with the goal of creating a sentence embedder. A further distinction to BERT is USE's unbounded input length, which lends itself well to the args.me corpus. The following pre-training tasks were considered: + +**- Skip-thought is a self-supervised pretraining task, originally devised to use LSTMs** + +to provide high-quality sentence vectors by training on a large amount of contiguous text [17]. + +**- Natural language response suggestion imparts conversational awareness to the sen-** + +tence encoder, which fits quite well to the task at hand. The goal of this supervised task is to predict the best short reply among millions of options in response to an email [15]. + +**- Stanford natural language inference is a labeled dataset [4] of 570,000 sentence** + +pairs. This can be seen as a supervised variant of BERT's next sentence prediction pre-training task. In this instance however, entailment, contradiction, and irrelevance are explicitly labeled in the data itself, rather than implied by the relative position of two sentences in an unlabeled corpus of contiguous text. These pre-training tasks make USE a great candidate for argument retrieval. Using USE, we embed each document in the args.me corpus into a 512-dimensional space. To retrieve arguments given a query, we embed the query using the same model into the same space, and perform exhaustive nearest-neighbor search,4 considering both L2 distance and inner-product (IP) distance for retrieval. We carried out a small pilot experiment to make sure USE projects the args.me corpus in a semantically meaningful way by running k-means on the embedded corpus, choosing a cluster size of 100. The clusters obtained are both syntactically and semantically coherent in a way that is surprisingly meaningful. Some clusters are thematically coherent, encompassing topics, such as religion, politics, and economics. Others are both syntactically and semantically coherent, where all premises are of the form "X is better than Y" and covering themes such as video game consoles, superheroes, and consumer electronics. Further clusters are only syntactically coherent, where, for instance, all arguments consist of YouTube links or of repeated short idiosyncratic phrases one tends to find on online debate websites (e.g., "I agree."). + +## 4 Evaluation + +The evaluation of our approaches to argument retrieval was carried out as part of the Touché shared task. In what follows, we briefly recap the experimental setup and overview the performance achieved. + +### 4.1 Experimental Setup + +The Touché shared task on argument retrieval [3] uses the TIRA evaluation platform [24] to judge entries to the competition. On TIRA every task participant is assigned their own virtual machine, and submitting a retrieval model to the shared task + +4 https://github.com/facebookresearch/faiss + +----- + +**Table 3. Evaluation results of our approaches compared to the two runner-ups of the shared task.** + +--- + +| Model | nDCG@5 | nDCG@10 | nDCG | QrelCoverage@10 | +|---|---|---|---|---| +| GPT-2 | 0.808 | 0.586 | 0.378 | 5.70 | +| Baseline (DirichletLM) | 0.756 | - | - | - | +| BERT | 0.755 | 0.538 | 0.337 | 5.36 | +| Team Aragorn | 0.684 | - | - | - | +| USE (L2) | 0.598 | 0.397 | 0.285 | 4.16 | +| Team Zorro | 0.573 | - | - | - | +| USE (IP) | 0.527 | 0.36 | 0.275 | 3.82 | + +--- + +corresponds to submitting software to be run on that virtual machine with the relevant inputs provided by TIRA at run time. Those inputs include the args.me corpus and a list of 50 topics (queries) on which the retrieval model is to be judged using crowdsourced relevance judgments. Though participant entries are ranked by nDCG@5 scores on the leaderboard,5 TIRA also returns nDCG@10, nDCG, and QrelCoverage@10, which we include in Table 3. + +### 4.2 Results + +The results of all our runs are included in Table 3, where one can clearly see the considerable improvement over the official baseline by our GPT-2 query expansion approach, which comes out on top of the leaderboard. Our BERT query expansion model basically ties with the baseline of 0.756 as it manages to score an nDCG@5 score of 0.755. We speculate that this performance might be partly due to the fact that both the args.me corpus and the datasets on which BERT and GPT-2 are trained consist of user-generated internet data. Both embedding-based runs perform worse than the query expansion approaches and that is to be expected, as the only information signal afforded to those runs originates in the query itself, whereas the other approaches had the benefit of considerable added context through query expansion. Still, judging the embeddings on their own constitutes a useful baseline. A promising approach would combine these two orthogonal approaches. + +## 5 Conclusion + +This work showcases three possible uses of transformer models for the retrieval of relevant arguments from the args.me corpus in particular, and document retrieval from a corpus in general. Impressive results were achieved without hyperparameter tuning or optimization of any sort. A promising future continuation of this work would be an ablation study that judges the effect that the hyperparameters have on retrieval-optimized text generation. Such a continuation would be necessary to judge the quantitative merits of each approach. + +5 https://events.webis.de/touche-20/shared-task-1.html + +----- + +It is also important to note that BERT and GPT-2 are merely the representatives of the transformer family of models. There exists a myriad of models6 that build on the foundations laid down by the works that introduced them, iterating, improving, and filling gaps those original models did not take into account. It would therefore be important to experiment with other models, perhaps also come up with new pre-training tasks that would make query expansion even more performant. Furthermore, our approach leaves out any sort of natural language preprocessing of the corpus or fine-tuning of any of the used models. That some approaches perform as well as they do is a testament to the amount of linguistic and world knowledge encoded in the weights of pre-trained transformers. A future research direction might leverage the args.me and Project Debater corpora to add more argumentative awareness to the transformers and indubitably improve retrieval results. Finally, we believe it crucial to sensibly modulate any research direction with the due ethical considerations of such projects. It is unclear whether to include user feedback, as including such signals would incur the risk of calcifying existing biases. While it might be useful to think of a search engine as an educational tool, it might prove dangerous to assume it is the prerogative of an information retrieval technology to monopolize the task of teaching its users how to think by conditioning them to blindly rely on it to populate their existing biases. + +## Bibliography + +[1] Ajjour, Y., Wachsmuth, H., Kiesel, J., Potthast, M., Hagen, M., Stein, B.: Data Acquisition + +for Argument Search: The args.me corpus. In: Benzmüller, C., Stuckenschmidt, H. (eds.) 42nd German Conference on Artificial Intelligence (KI 2019), pp. 48-59, Springer, Berlin Heidelberg New York (Sep 2019), https://doi.org/10.1007/978-3-030-30179-8\_4 [2] Azad, H.K., Deepak, A.: Query expansion techniques for information retrieval: A survey. + +Information Processing & Management 56(5), 1698?1735 (Sep 2019), ISSN 0306-4573, https://doi.org/10.1016/j.ipm.2019.05.009, URL http://dx.doi.org/10.1016/j.ipm.2019.05.009 [3] Bondarenko, A., Fröbe, M., Beloucif, M., Gienapp, L., Ajjour, Y., Panchenko, A., + +Biemann, C., Stein, B., Wachsmuth, H., Potthast, M., Hagen, M.: Overview of Touché 2020: Argument Retrieval. In: Working Notes Papers of the CLEF 2020 Evaluation Labs (Sep 2020), ISSN 1613-0073 [4] Bowman, S.R., Angeli, G., Potts, C., Manning, C.D.: A large annotated corpus for learning + +natural language inference. In: Màrquez, L., Callison-Burch, C., Su, J., Pighin, D., Marton, Y. (eds.) Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing, EMNLP 2015, Lisbon, Portugal, September 17-21, 2015, pp. 632-642, The Association for Computational Linguistics (2015), https://doi.org/10.18653/v1/d15-1075, URL https://doi.org/10.18653/v1/d15-1075 [5] Cer, D., Yang, Y., Kong, S., Hua, N., Limtiaco, N., John, R.S., Constant, N., + +Guajardo-Cespedes, M., Yuan, S., Tar, C., Sung, Y., Strope, B., Kurzweil, R.: Universal sentence encoder. CoRR abs/1803.11175 (2018), URL http://arxiv.org/abs/1803.11175 [6] Chakrabarty, T., Hidey, C., Muresan, S., McKeown, K., Hwang, A.: AMPERSAND: + +argument mining for persuasive online discussions. In: Inui, K., Jiang, J., Ng, V., Wan, X. 6 https://github.com/thunlp/PLMpapers + +----- + +(eds.) Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing, EMNLP-IJCNLP 2019, Hong Kong, China, November 3-7, 2019, pp. 2933-2943, Association for Computational Linguistics (2019), https://doi.org/10.18653/v1/D19-1291, URL https://doi.org/10.18653/v1/D19-1291 [7] Devlin, J., Chang, M., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional + +transformers for language understanding. CoRR abs/1810.04805 (2018), URL http://arxiv.org/abs/1810.04805 [8] Dibia, V.: Neuralqa: A usable library for question answering (contextual query expansion + ++ bert) on large datasets (2020) [9] Ein-Dor, L., Shnarch, E., Dankin, L., Halfon, A., Sznajder, B., Gera, A., Alzate, C., Gleize, + +M., Choshen, L., Hou, Y., Bilu, Y., Aharonov, R., Slonim, N.: Corpus wide argument mining - A working solution. In: The Thirty-Fourth AAAI Conference on Artificial Intelligence, AAAI 2020, The Thirty-Second Innovative Applications of Artificial Intelligence Conference, IAAI 2020, The Tenth AAAI Symposium on Educational Advances in Artificial Intelligence, EAAI 2020, New York, NY, USA, February 7-12, 2020, pp. 7683-7691, AAAI Press (2020), URL https://aaai.org/ojs/index.php/AAAI/article/view/6270 [10] Fedus, W., Goodfellow, I.J., Dai, A.M.: Maskgan: Better text generation via filling in the + +\_\_\_\_\_\_\_. In: 6th International Conference on Learning Representations, ICLR 2018, Vancouver, BC, Canada, April 30 - May 3, 2018, Conference Track Proceedings, OpenReview.net (2018), URL https://openreview.net/forum?id=ByOExmWAb [11] Fromm, M., Faerman, E., Seidl, T.: TACAM: topic and context aware argument mining. + +In: Barnaghi, P.M., Gottlob, G., Manolopoulos, Y., Tzouramanis, T., Vakali, A. (eds.) 2019 IEEE/WIC/ACM International Conference on Web Intelligence, WI 2019, Thessaloniki, Greece, October 14-17, 2019, pp. 99-106, ACM (2019), https://doi.org/10.1145/3350546.3352506, URL https://doi.org/10.1145/3350546.3352506 [12] Géron, A.: Hands-on machine learning with Scikit-Learn, Keras, and TensorFlow: + +Concepts, tools, and techniques to build intelligent systems. O'Reilly Media (2019) [13] Goodfellow, I., Bengio, Y., Courville, A., Bengio, Y.: Deep learning, vol. 1. MIT press + +Cambridge (2016), ISBN 978-0-262-03561-3 [14] Gretz, S., Friedman, R., Cohen-Karlik, E., Toledo, A., Lahav, D., Aharonov, R., Slonim, + +N.: A large-scale dataset for argument quality ranking: Construction and analysis. In: The Thirty-Fourth AAAI Conference on Artificial Intelligence, AAAI 2020, The Thirty-Second Innovative Applications of Artificial Intelligence Conference, IAAI 2020, The Tenth AAAI Symposium on Educational Advances in Artificial Intelligence, EAAI 2020, New York, NY, USA, February 7-12, 2020, pp. 7805-7813, AAAI Press (2020), URL https://aaai.org/ojs/index.php/AAAI/article/view/6285 [15] Henderson, M.L., Al-Rfou, R., Strope, B., Sung, Y., Lukács, L., Guo, R., Kumar, S., + +Miklos, B., Kurzweil, R.: Efficient natural language response suggestion for smart reply. CoRR abs/1705.00652 (2017), URL http://arxiv.org/abs/1705.00652 [16] Holtzman, A., Buys, J., Du, L., Forbes, M., Choi, Y.: The curious case of neural text + +degeneration. In: 8th International Conference on Learning Representations, ICLR 2020, Addis Ababa, Ethiopia, April 26-30, 2020, OpenReview.net (2020), URL https://openreview.net/forum?id=rygGQyrFvH [17] Kiros, R., Zhu, Y., Salakhutdinov, R., Zemel, R.S., Urtasun, R., Torralba, A., Fidler, S.: + +Skip-thought vectors. In: Cortes, C., Lawrence, N.D., Lee, D.D., Sugiyama, M., Garnett, R. (eds.) Advances in Neural Information Processing Systems 28: Annual Conference on Neural Information Processing Systems 2015, December 7-12, 2015, Montreal, Quebec, Canada, pp. 3294-3302 (2015), URL http://papers.nips.cc/paper/5950-skip-thought-vectors + +----- + +[18] Lawrence, J., Reed, C.: Argument mining: A survey. Computational Linguistics 45(4), + +765-818 (2020), https://doi.org/10.1162/coli\_a\_00364, URL https://doi.org/10.1162/coli\_a\_00364 [19] Lin, S.C., Yang, J.H., Nogueira, R., Tsai, M.F., Wang, C.J., Lin, J.: Query reformulation + +using query history for passage retrieval in conversational search (2020) [20] Maaten, L.v.d., Hinton, G.: Visualizing data using t-sne. Journal of machine learning + +research 9(Nov), 2579-2605 (2008) [21] Naseri, S., Dalton, J.S., Allan, J., Yates, A.: Ceqe: Contextualized embeddings forquery + +expansion (2020) [22] Padaki, R., Dai, Z., Callan, J.: Rethinking query expansion for bert reranking. In: European + +Conference on Information Retrieval, pp. 297-304, Springer (2020) [23] Potthast, M., Gienapp, L., Euchner, F., Heilenkötter, N., Weidmann, N., Wachsmuth, H., + +Stein, B., Hagen, M.: Argument search: Assessing argument relevance. In: Piwowarski, B., Chevalier, M., Gaussier, É., Maarek, Y., Nie, J., Scholer, F. (eds.) Proceedings of the 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2019, Paris, France, July 21-25, 2019, pp. 1117-1120, ACM (2019), https://doi.org/10.1145/3331184.3331327, URL https://doi.org/10.1145/3331184.3331327 [24] Potthast, M., Gollub, T., Wiegmann, M., Stein, B.: TIRA integrated research architecture. + +In: Ferro, N., Peters, C. (eds.) Information Retrieval Evaluation in a Changing World - Lessons Learned from 20 Years of CLEF, The Information Retrieval Series, vol. 41, pp. 123-160, Springer (2019), https://doi.org/10.1007/978-3-030-22948-1\_5, URL https://doi.org/10.1007/978-3-030-22948-1\\\_5 [25] Radford, A., Narasimhan, K., Salimans, T., Sutskever, I.: Improving language + +understanding by generative pre-training (2018) [26] Rahwan, I., Zablith, F., Reed, C.: Laying the foundations for a world wide argument web. + +Artificial Intelligence 171(10), 897-921 (2007), ISSN 0004-3702, https://doi.org/https://doi.org/10.1016/j.artint.2007.04.015 [27] Reed, C. (ed.): Proceedings of the Third Workshop on Argument Mining + +(ArgMining2016), Association for Computational Linguistics, Berlin, Germany (Aug 2016), https://doi.org/10.18653/v1/W16-28, URL https://www.aclweb.org/anthology/W16-2800 [28] Teufel, S., et al.: Argumentative zoning: Information extraction from scientific text. Ph.D. + +thesis, Citeseer (1999) [29] Toledo, A., Gretz, S., Cohen-Karlik, E., Friedman, R., Venezian, E., Lahav, D., Jacovi, M., + +Aharonov, R., Slonim, N.: Automatic argument quality assessment - new datasets and methods. In: Inui, K., Jiang, J., Ng, V., Wan, X. (eds.) Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing, EMNLP-IJCNLP 2019, Hong Kong, China, November 3-7, 2019, pp. 5624-5634, Association for Computational Linguistics (2019), https://doi.org/10.18653/v1/D19-1564, URL https://doi.org/10.18653/v1/D19-1564 [30] Vaswani, A., Shazeer, N., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A.N., Kaiser, L., + +Polosukhin, I.: Attention is all you need. In: Guyon, I., von Luxburg, U., Bengio, S., Wallach, H.M., Fergus, R., Vishwanathan, S.V.N., Garnett, R. (eds.) Advances in Neural Information Processing Systems 30: Annual Conference on Neural Information Processing Systems 2017, 4-9 December 2017, Long Beach, CA, USA, pp. 5998-6008 (2017), URL http://papers.nips.cc/paper/7181-attention-is-all-you-need [31] Wachsmuth, H., Potthast, M., Khatib, K.A., Ajjour, Y., Puschmann, J., Qu, J., Dorsch, J., + +Morari, V., Bevendorff, J., Stein, B.: Building an argument search engine for the web. In: Habernal, I., Gurevych, I., Ashley, K.D., Cardie, C., Green, N., Litman, D.J., Petasis, G., Reed, C., Slonim, N., Walker, V.R. (eds.) Proceedings of the 4th Workshop on Argument + +----- + +Mining, ArgMining@EMNLP 2017, Copenhagen, Denmark, September 8, 2017, pp. 49-59, Association for Computational Linguistics (2017), https://doi.org/10.18653/v1/w17-5106, URL https://doi.org/10.18653/v1/w17-5106 [32] Wolf, T., Debut, L., Sanh, V., Chaumond, J., Delangue, C., Moi, A., Cistac, P., Rault, T., + +Louf, R., Funtowicz, M., Brew, J.: Huggingface's transformers: State-of-the-art natural language processing. CoRR abs/1910.03771 (2019), URL http://arxiv.org/abs/1910.03771 [33] Yang, W., Xie, Y., Lin, A., Li, X., Tan, L., Xiong, K., Li, M., Lin, J.: End-to-end + +open-domain question answering with bertserini. In: Ammar, W., Louis, A., Mostafazadeh, N. (eds.) Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, NAACL-HLT 2019, Minneapolis, MN, USA, June 2-7, 2019, Demonstrations, pp. 72-77, Association for Computational Linguistics (2019), https://doi.org/10.18653/v1/n19-4013, URL https://doi.org/10.18653/v1/n19-4013 [34] Yang, W., Zhang, H., Lin, J.: Simple applications of BERT for ad hoc document retrieval. + +CoRR abs/1903.10972 (2019), URL http://arxiv.org/abs/1903.10972 [35] Yilmaz, Z.A., Wang, S., Yang, W., Zhang, H., Lin, J.: Applying BERT to document + +retrieval with birch. In: Padó, S., Huang, R. (eds.) Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing, EMNLP-IJCNLP 2019, Hong Kong, China, November 3-7, 2019 - System Demonstrations, pp. 19-24, Association for Computational Linguistics (2019), https://doi.org/10.18653/v1/D19-3004, URL https://doi.org/10.18653/v1/D19-3004 [36] Zhai, C., Lafferty, J.: A study of smoothing methods for language models applied to ad hoc + +information retrieval. In: Proceedings of the 24th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 334-342, SIGIR '01, Association for Computing Machinery, New York, NY, USA (2001), ISBN 1581133316, https://doi.org/10.1145/383952.384019, URL https://doi.org/10.1145/383952.384019 \ No newline at end of file diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/tables/page004-table01.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/tables/page004-table01.png new file mode 100644 index 0000000..f63d221 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/tables/page004-table01.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/tables/page006-table01.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/tables/page006-table01.png new file mode 100644 index 0000000..6ea2f37 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/tables/page006-table01.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/tables/page008-table01.png b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/tables/page008-table01.png new file mode 100644 index 0000000..09146c2 Binary files /dev/null and b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/tables/page008-table01.png differ diff --git a/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/tables/table-01.txt.md b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/tables/table-01.txt.md new file mode 100644 index 0000000..48015d3 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpora/touche-20-task-1-spot-check/papers/241/tables/table-01.txt.md @@ -0,0 +1,9 @@ +| Model | nDCG@5 | nDCG@10 | nDCG | QrelCoverage@10 | +|---|---|---|---|---| +| GPT-2 | 0.808 | 0.586 | 0.378 | 5.70 | +| Baseline (DirichletLM) | 0.756 | - | - | - | +| BERT | 0.755 | 0.538 | 0.337 | 5.36 | +| Team Aragorn | 0.684 | - | - | - | +| USE (L2) | 0.598 | 0.397 | 0.285 | 4.16 | +| Team Zorro | 0.573 | - | - | - | +| USE (IP) | 0.527 | 0.36 | 0.275 | 3.82 | diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/README.md b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/README.md new file mode 100644 index 0000000..b099789 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/README.md @@ -0,0 +1,292 @@ +# To incorporate + +the data is moved to `/mnt/ceph/storage/data-in-progress/data-research/agentic-ai/INLG-27-corpus` + +# Shared-Task Corpus — Build, Layout, and Usage + +A corpus of CLEF shared tasks where each entry links one **overview paper** (written by the +lab organizers, summarising the whole task) to the **notebook papers** written by the teams +that participated in it. + +The intended use is generation: the notebook papers are the inputs, the overview paper is +the target output. + +**42 tasks · 444 notebook papers · 486 parsed documents · CLEF 2018–2025 · 22 labs** + +--- + +## 1. Where the data comes from + +Everything derives from two public sources, both fetched once and cached: + +| Source | Role | +|---|---| +| [CEUR-WS](https://ceur-ws.org) volume index pages | Authoritative table of contents: which papers exist, in which lab section, in what order | +| [DBLP](https://dblp.org) working-notes records | Cross-check only — confirms titles and supplies cleaner author name spellings | + +Eight volumes, one per CLEF edition: 2125 (2018), 2380, 2696, 2936, 3180, 3497, 3740, +4038 (2025). + +DBLP never adds or removes a paper. CEUR-WS is authoritative for *which* papers exist and +what section they sit in; DBLP only corrects author spellings where the normalized titles +match (99%+ of papers). + +**Every `pdf_url` in the corpus was read from an `href` on a CEUR-WS index page.** None are +constructed from a filename pattern — the patterns are not stable across volumes +(`paper-199.pdf` in Vol-3497, `paper_281.pdf` in Vol-4038), so guessing them silently +produces dead links. + +--- + +## 2. How tasks were identified + +This is the part worth understanding before you trust an entry, because it is where the +judgement lives. + +A CEUR volume is organised into lab sections (PAN, Touché, eRisk, …). Within a section we +need to know which overview paper each notebook paper belongs to. Two cases: + +**One overview in the section** → every other paper in that section is its participant. +This is structural, derived from the published table of contents, and is the only case +included in the released corpus. Marked `confidence: "high"`, +`task_assignment_method: "section_grouping"`. + +**Several overviews in the section** → assignment falls back to matching notebook titles +against overview titles. This is *not* reliable, so those entries are marked +`confidence: "medium"` and written to `data/intermediate/needs_review.jsonl` for human +review. **156 of 198 candidate tasks are in this state and are excluded from the released +corpus.** + +> The original plan assumed overviews and their participants appear contiguously, so +> positional grouping would work throughout. They do not: CEUR volumes list *all* of a +> lab's overviews first, then the notebook papers, in an order that is neither per-task +> nor alphabetical. That is why multi-overview sections need review rather than trust. + +### Two failure modes that were found and fixed + +Both were caught by reading the data rather than by tests passing, and both are worth +knowing about if you extend this to other venues: + +- **Organizer papers that never say "Overview".** The ELOQUENT 2024 section publishes three + organizer task papers, but only one has "Overview" in its title. The other two + (`ELOQUENT 2024 — Topical Quiz Task`, `— Robustness Task`) were initially filed as + participant submissions. Detection now also matches the lab-branded + ` — Task` shape. +- **A hidden second task.** CLEF eHealth 2021 has two real overviews, but the second is + titled `Consumer Health Search at CLEF eHealth 2021` — invisible to any keyword rule. + The section looked single-overview and was wrongly trusted. Now, if the detected overview + names its own task number and a "participant" names a *different* one, the task is + downgraded to review rather than trusted. + +An author-overlap rule was tried for both and **rejected**: lab organizers routinely also +submit competing systems (`DPRL Systems in the CLEF 2021 ARQMath Lab`, `Organiser Team at +ImageCLEFlifelog 2020`), so it flagged ~48 genuine participant papers. + +--- + +## 3. Files and layout + +``` +data/final/ +├── shared_tasks.jsonl one JSON record per task — the primary artifact +├── shared_tasks.csv the same, flattened one row per task, for spreadsheets +├── report.md generated summary: counts, coverage, link stats +└── fulltext/ + ├── README.md + ├── manifest.jsonl one record per parsed document + └── {task_id}/ + ├── overview.md the target output + ├── participants/{paper_stem}.md the inputs + ├── figures/{doc}/img_p4_1.png figures, per document + └── tables/{doc}/ + ├── table-01.md table as parsed text + └── page011-table01.png table cropped from the page +``` + +`{doc}` is `overview` or a notebook paper's stem. `{paper_stem}` matches the source PDF +filename on CEUR-WS, so any document traces back to its origin. + +The raw PDFs are **not** in git — they are re-fetchable (see §7) and were removed to keep +the repository usable. + +### Task record + +```json +{ + "task_id": "clef2020-touch-touch-2020-argument-retrieval", + "venue": "Touché", "parent_venue": "CLEF", "year": 2020, + "task_name": "Touché 2020: Argument Retrieval", + "ceur_volume": "2696", + "overview": { + "title": "Overview of Touché 2020: Argument Retrieval", + "pdf_url": "https://ceur-ws.org/Vol-2696/paper_261.pdf", + "authors": ["Alexander Bondarenko", "..."], + "is_umbrella": false, + "fulltext_path": "data/final/fulltext/clef2020-.../overview.md", + "figures_dir": "...", "n_figures": 0, + "tables_dir": "...", "n_tables": 6 + }, + "participants": [ + { + "title": "An Open-Domain Web Search Engine for Answering Comparative Questions", + "authors": ["..."], + "pdf_url": "https://ceur-ws.org/Vol-2696/paper_130.pdf", + "team_name": null, + "fulltext_path": "data/final/fulltext/clef2020-.../participants/paper_130.md", + "code_urls": ["https://github.com/hemiipatu/Blocklists.git"], + "code_url_details": [{"url": "...", "status": "200", "availability_evidence": false}], + "third_party_urls": ["https://github.com/huggingface/transformers"], + "tira_refs": [], "n_figures": 1, "n_tables": 1 + } + ], + "counts": { + "notebook_papers": 10, + "teams_claimed_in_overview": 17, + "runs_claimed_in_overview": 41, + "coverage_ratio": 0.588 + }, + "provenance": { + "task_assignment_method": "section_grouping", + "confidence": "high", + "extracted_at": "2026-08-16" + } +} +``` + +In the CSV, `participant_pdf_urls` and `participant_fulltext_paths` are joined by `; ` in +the **same order**, so the columns align positionally. + +--- + +## 4. Field notes + +**`coverage_ratio`** = `notebook_papers / teams_claimed_in_overview`. Not every team that +competes writes a paper, so a ratio below 1 is normal and expected, not a bug — Touché 2020 +reports 17 teams and 41 runs but published 10 notebook papers. **It is `null` for 21 of 42 +tasks**, where the overview does not state a participation count in extractable prose. Team +counts are only taken from statements about *actual participation*; registration counts +("98 teams registered") are deliberately refused, since they would inflate the ratio. + +**`code_urls`** contains links plausibly pointing at the *team's own* code. Dependencies the +team merely used are excluded into `third_party_urls`. This distinction matters: before it +existed, ~48% of stored links were things like `huggingface/transformers`, `nltk`, and +`meta-llama` presented as team submissions. Links are never dropped for being dead — a dead +repository is still evidence the team published code — so filter on +`code_url_details[].status` yourself. `availability_evidence: true` marks links that +appeared beside an explicit code-release statement. + +**`is_umbrella`** is `true` when one overview serves several sub-tasks. Best-effort: a lab +whose titles omit task numbers entirely reads as non-umbrella even if it ran several. + +**`team_name`** is extracted only from the two attribution shapes CEUR titles actually use +(`TEAM at Venue Year: …`, `TEAM@Venue: …`), and is `null` rather than guessed otherwise — +so expect it on roughly half of participants (205/444). + +--- + +## 5. Full text, figures, and tables + +Text comes from each PDF's own text layer via [liteparse](https://github.com/run-llama/liteparse), +output as Markdown to preserve heading structure. 18.4M characters across 486 documents. + +**OCR is not used, and does not need to be.** Measured across all 504 PDFs: 0 are garbled, +0 are scanned page images, and exactly **1** lacks a usable text layer +(`Vol-3740/paper-124.pdf`, whose text is drawn as vector outlines). It is flagged +`needs_ocr` in `manifest.jsonl`. If you want it, liteparse delegates OCR over HTTP, so serve +a model and point at it: + +```bash +./src/parse_fulltext.py --ocr-server-url http://localhost:8080 --only-needs-ocr +``` + +**Figures** (1,291) are the raster images embedded in the PDFs, referenced inline from the +markdown so a document still reads as a whole. Figures drawn as *vector* graphics — many +plots and diagrams — are not files and are not extracted; their captions remain in the text. + +**Tables** exist in two independent views, and this distinction matters: + +| View | Count | How it is produced | +|---|---|---| +| `table-NN.md` | 3,199 | The parser's text reconstruction, in document order | +| `pageNNN-tableNN.png` | 3,055 | Cropped from the page using the paper's own ruling lines | + +**They are not index-matched, and where they disagree, trust the image.** The text +reconstruction is unreliable for large tables — in the eRisk 2018 overview the parser +collapsed a 34-team results table into a single markdown row. Pairing images to markdown +tables by matching cell text was tried and produced images filed under the wrong table, so +images are now named for the page they came from and always show what they claim to. + +--- + +## 6. Using it + +```python +import json +from pathlib import Path + +tasks = [json.loads(l) for l in open("data/final/shared_tasks.jsonl")] + +for task in tasks: + target = Path(task["overview"]["fulltext_path"]).read_text() + inputs = [Path(p["fulltext_path"]).read_text() for p in task["participants"]] + # inputs -> target +``` + +Filter to the best-evidenced entries: + +```python +solid = [t for t in tasks + if t["counts"]["coverage_ratio"] and t["counts"]["coverage_ratio"] >= 0.7 + and not t["overview"]["is_umbrella"]] +``` + +> **Before you build a test split:** the overview papers are the target output, and their +> full text ships inside this corpus. Any blind evaluation split must withhold +> `overview.md`, or the answer leaks. + +--- + +## 7. Reproducing + +Setup — note the parser is an npm package, so `pip install` alone is not enough: + +```bash +pyenv activate uniagent +pip install -r requirements.txt +npm i -g @llamaindex/liteparse # provides the `lit` command +``` + +Every stage is independently re-runnable and caches to disk; nothing re-fetches what is +already there. Run from the project root: + +```bash +./src/fetch_volumes.py # CEUR + DBLP pages -> data/raw/ +./src/parse_sections.py # sections and papers -> data/intermediate/sections/ +./src/group_tasks.py # tasks -> data/intermediate/all_candidates.jsonl +./src/extract_counts.py # claimed team/run counts +./src/find_code.py # code + TIRA links (slow: fetches every PDF) +./src/parse_fulltext.py # markdown, figures, tables +./src/build_corpus.py # assemble data/final/ +``` + +`build_corpus.py` validates before writing anything, and refuses to emit the corpus if a +check fails: every task has exactly one overview and ≥1 participant, no duplicate `task_id` +or `pdf_url`, and every `coverage_ratio` is null or within `[0, 1.5]`. + +`pytest` covers the parsing and grouping logic against saved fixtures — 68 tests, no +network. + +--- + +## 8. Known limitations + +1. **42 of 198 candidate tasks are released.** The other 156 need human review of their + overview→participant assignment (§2) and sit in `data/intermediate/needs_review.jsonl`. +2. **`coverage_ratio` is unknown for half the corpus** (21/42), so the plan's intended + quality filter cannot be applied everywhere. +3. **Vector figures are not extracted** — only raster images are, which is why 267 of 486 + documents have figure files rather than nearly all. +4. **Markdown tables are unreliable for large tables.** Use the images. +5. **CLEF only.** SemEval, standalone PAN, and Touché editions outside CLEF are not + included; the target of 30–50 tasks was met without them. +6. **One document lacks a usable text layer** and needs OCR to be complete (§5). diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/logs/.gitkeep b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/notebooks/.gitkeep b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/notebooks/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/pytest.ini b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/pytest.ini new file mode 100644 index 0000000..a635c5c --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +pythonpath = . diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/requirements.txt b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/requirements.txt new file mode 100644 index 0000000..03c7f13 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/requirements.txt @@ -0,0 +1,5 @@ +beautifulsoup4==4.15.0 +lxml==6.1.1 +pillow==12.3.0 +pytest==9.1.1 +requests==2.34.2 diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/build_corpus.py b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/build_corpus.py new file mode 100755 index 0000000..10078ec --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/build_corpus.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python +"""Assemble the final benchmark deliverables from the high-confidence candidate tasks, +joined with Stage 4 counts and Stage 5 code links. See PLAN.md sections 1, 5, 6.""" +import argparse +import csv +import json +import logging +import random +import sys +from collections import Counter +from datetime import datetime +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +CANDIDATES_PATH = PROJECT_ROOT / "data" / "intermediate" / "all_candidates.jsonl" +COUNTS_DIR = PROJECT_ROOT / "data" / "intermediate" / "counts" +CODE_DIR = PROJECT_ROOT / "data" / "intermediate" / "code" +FINAL_DIR = PROJECT_ROOT / "data" / "final" +FULLTEXT_MANIFEST_PATH = FINAL_DIR / "fulltext" / "manifest.jsonl" +LOGS_DIR = PROJECT_ROOT / "logs" + +CSV_FIELDS = [ + "task_id", "venue", "parent_venue", "year", "task_name", "ceur_volume", + "overview_title", "overview_pdf_url", "overview_authors", "is_umbrella", + "notebook_papers", "teams_claimed_in_overview", "runs_claimed_in_overview", "coverage_ratio", + "participant_pdf_urls", "team_names", "overview_fulltext_path", "participant_fulltext_paths", + "code_urls", "code_urls_live", "tira_refs", + "task_assignment_method", "confidence", "extracted_at", +] + + +def setup_logging() -> Path: + LOGS_DIR.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_path = LOGS_DIR / f"build_corpus_{timestamp}.log" + + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") + + file_handler = logging.FileHandler(log_path, mode="w") + file_handler.setFormatter(formatter) + root_logger.addHandler(file_handler) + + stream_handler = logging.StreamHandler(sys.stderr) + stream_handler.setFormatter(formatter) + root_logger.addHandler(stream_handler) + + return log_path + + +def join_counts(task: dict, logger: logging.Logger) -> None: + counts_path = COUNTS_DIR / f"{task['task_id']}.json" + teams = runs = None + if counts_path.exists(): + data = json.loads(counts_path.read_text(encoding="utf-8")) + teams, runs = data.get("teams"), data.get("runs") + else: + logger.warning("%s: no counts file found — coverage_ratio will be null", task["task_id"]) + + notebook_papers = len(task["participants"]) + coverage_ratio = round(notebook_papers / teams, 3) if teams else None + + task["counts"] = { + "notebook_papers": notebook_papers, + "teams_claimed_in_overview": teams, + "runs_claimed_in_overview": runs, + "coverage_ratio": coverage_ratio, + } + + +def join_code_links(task: dict, logger: logging.Logger) -> None: + code_path = CODE_DIR / f"{task['task_id']}.json" + if not code_path.exists(): + logger.warning("%s: no code-links file found — code_urls/tira_refs left empty", task["task_id"]) + return + + by_pdf_url = {entry["pdf_url"]: entry for entry in json.loads(code_path.read_text(encoding="utf-8"))} + for participant in task["participants"]: + entry = by_pdf_url.get(participant["pdf_url"]) + if entry is None: + continue + # PLAN.md section 1 keeps code_urls a flat URL list. A dead link is still evidence + # the team had a repo, so nothing is filtered by HTTP status — but the status and + # the code-availability evidence are carried alongside in code_url_details so a + # consumer can tell a live repo from a 404 or an unverified rate-limited check + # without re-fetching every link. + participant["code_urls"] = [c["url"] for c in entry["code_urls"]] + participant["code_url_details"] = [ + {"url": c["url"], "status": c["status"], "availability_evidence": c.get("evidence", False)} + for c in entry["code_urls"] + ] + participant["third_party_urls"] = entry.get("third_party_urls", []) + participant["tira_refs"] = entry["tira_refs"] + + +def join_fulltext_paths(task: dict, manifest: dict[str, str], logger: logging.Logger) -> None: + """Publish each document's parsed-markdown path on the record itself, so consumers can + go from a corpus entry straight to its text without deriving filenames from URLs.""" + if not manifest: + return + + def apply(document: dict, record: dict | None) -> None: + document["fulltext_path"] = record["markdown_path"] if record else None + document["figures_dir"] = record.get("figures_dir") if record else None + document["n_figures"] = record.get("n_figures", 0) if record else 0 + document["tables_dir"] = record.get("tables_dir") if record else None + document["n_tables"] = record.get("n_tables", 0) if record else 0 + + overview_record = manifest.get(task["overview"]["pdf_url"]) + apply(task["overview"], overview_record) + if overview_record is None: + logger.warning("%s: overview has no parsed full text", task["task_id"]) + + missing = 0 + for participant in task["participants"]: + record = manifest.get(participant["pdf_url"]) + apply(participant, record) + if record is None: + missing += 1 + if missing: + logger.warning("%s: %d/%d participants have no parsed full text", task["task_id"], missing, len(task["participants"])) + + +def load_fulltext_manifest(logger: logging.Logger) -> dict[str, dict]: + """Map pdf_url -> parsed-document record from Stage 7's manifest, if it has run.""" + if not FULLTEXT_MANIFEST_PATH.exists(): + logger.warning("no full-text manifest at %s — run parse_fulltext.py to add fulltext_path fields", FULLTEXT_MANIFEST_PATH) + return {} + manifest = {} + for line in FULLTEXT_MANIFEST_PATH.read_text(encoding="utf-8").splitlines(): + record = json.loads(line) + manifest[record["pdf_url"]] = record + logger.info("loaded full-text manifest with %d documents", len(manifest)) + return manifest + + +CONFIDENCE_RANK = {"high": 0, "medium": 1, "low": 2} + + +def selection_key(task: dict) -> tuple: + """PLAN.md section 3 Stage 6 ranking: coverage_ratio descending, then high confidence + before medium, then non-umbrella before umbrella, then more participants first. + + A null coverage_ratio means the overview's team count was not extractable, so it + sorts below every known ratio rather than being treated as a zero or a perfect score.""" + ratio = task["counts"]["coverage_ratio"] + return ( + 0 if ratio is not None else 1, + -(ratio if ratio is not None else 0), + CONFIDENCE_RANK.get(task["provenance"]["confidence"], 9), + 1 if task["overview"]["is_umbrella"] else 0, + -len(task["participants"]), + ) + + +def select_corpus(tasks: list[dict], target: int, logger: logging.Logger) -> list[dict]: + """Rank candidates and emit the top `target`. The selection is a view, not a + destructive filter — every candidate remains in all_candidates.jsonl (PLAN.md).""" + ranked = sorted(tasks, key=selection_key) + if len(ranked) > target: + logger.info("selecting top %d of %d ranked candidates", target, len(ranked)) + return ranked[:target] + logger.info("all %d candidates are within the target of %d; emitting all, ranked", len(ranked), target) + return ranked + + +def validate(tasks: list[dict], logger: logging.Logger) -> bool: + ok = True + + for task in tasks: + if not task["participants"]: + logger.error("VALIDATION FAILED: %s has 0 participants", task["task_id"]) + ok = False + + task_ids = [t["task_id"] for t in tasks] + duplicate_ids = {tid for tid in task_ids if task_ids.count(tid) > 1} + if duplicate_ids: + logger.error("VALIDATION FAILED: duplicate task_id(s): %s", duplicate_ids) + ok = False + + pdf_urls: dict[str, str] = {} + for t in tasks: + urls = [t["overview"]["pdf_url"]] + [p["pdf_url"] for p in t["participants"]] + for url in urls: + if url in pdf_urls and pdf_urls[url] != t["task_id"]: + logger.error("VALIDATION FAILED: duplicate pdf_url %s across %s and %s", url, pdf_urls[url], t["task_id"]) + ok = False + pdf_urls[url] = t["task_id"] + + for t in tasks: + ratio = t["counts"]["coverage_ratio"] + if ratio is not None and not (0 <= ratio <= 1.5): + logger.error("VALIDATION FAILED: %s coverage_ratio=%.3f outside [0, 1.5]", t["task_id"], ratio) + ok = False + + sample = random.sample(tasks, min(5, len(tasks))) + logger.info("--- spot-check sample (%d tasks) ---", len(sample)) + for t in sample: + logger.info("%s", t["task_id"]) + logger.info(" overview: %s", t["overview"]["title"]) + for p in t["participants"][:5]: + logger.info(" - %s", p["title"]) + if len(t["participants"]) > 5: + logger.info(" ... and %d more", len(t["participants"]) - 5) + + return ok + + +def write_jsonl(tasks: list[dict], path: Path) -> None: + with path.open("w", encoding="utf-8") as f: + for t in tasks: + f.write(json.dumps(t, ensure_ascii=False) + "\n") + + +def write_csv(tasks: list[dict], path: Path) -> None: + with path.open("w", encoding="utf-8", newline="") as f: + writer = csv.DictWriter(f, fieldnames=CSV_FIELDS) + writer.writeheader() + for t in tasks: + all_code_urls = sorted({u for p in t["participants"] for u in p["code_urls"]}) + live_code_urls = sorted({ + d["url"] for p in t["participants"] for d in p.get("code_url_details", []) + if d["status"] == "200" + }) + team_names = sorted({p["team_name"] for p in t["participants"] if p["team_name"]}) + all_tira_refs = sorted({r for p in t["participants"] for r in p["tira_refs"]}) + writer.writerow({ + "task_id": t["task_id"], + "venue": t["venue"], + "parent_venue": t["parent_venue"], + "year": t["year"], + "task_name": t["task_name"], + "ceur_volume": t["ceur_volume"], + "overview_title": t["overview"]["title"], + "overview_pdf_url": t["overview"]["pdf_url"], + "overview_authors": "; ".join(t["overview"]["authors"]), + "is_umbrella": t["overview"]["is_umbrella"], + "notebook_papers": t["counts"]["notebook_papers"], + "teams_claimed_in_overview": t["counts"]["teams_claimed_in_overview"], + "runs_claimed_in_overview": t["counts"]["runs_claimed_in_overview"], + "coverage_ratio": t["counts"]["coverage_ratio"], + "participant_pdf_urls": "; ".join(p["pdf_url"] for p in t["participants"]), + "team_names": "; ".join(team_names), + "overview_fulltext_path": t["overview"].get("fulltext_path") or "", + # Positionally aligned with participant_pdf_urls; empty where unparsed. + "participant_fulltext_paths": "; ".join(p.get("fulltext_path") or "" for p in t["participants"]), + "code_urls": "; ".join(all_code_urls), + "code_urls_live": "; ".join(live_code_urls), + "tira_refs": "; ".join(all_tira_refs), + "task_assignment_method": t["provenance"]["task_assignment_method"], + "confidence": t["provenance"]["confidence"], + "extracted_at": t["provenance"]["extracted_at"], + }) + + +def write_report(tasks: list[dict], path: Path) -> None: + by_venue_year = Counter((t["parent_venue"], t["year"]) for t in tasks) + ratios = [t["counts"]["coverage_ratio"] for t in tasks if t["counts"]["coverage_ratio"] is not None] + total_participants = sum(len(t["participants"]) for t in tasks) + with_code = sum(1 for t in tasks for p in t["participants"] if p["code_urls"] or p["tira_refs"]) + unresolved = total_participants - with_code + + lines = [ + "# Shared-Task Corpus Report", + "", + f"Generated: {datetime.now().date().isoformat()}", + f"Total tasks: {len(tasks)}", + "", + "## Tasks per venue/year", + "", + "| Parent venue | Year | Tasks |", + "|---|---|---|", + ] + for (venue, year), count in sorted(by_venue_year.items(), key=lambda kv: (kv[0][0], kv[0][1])): + lines.append(f"| {venue} | {year} | {count} |") + + umbrella = sum(1 for t in tasks if t["overview"]["is_umbrella"]) + named_teams = sum(1 for t in tasks for p in t["participants"] if p["team_name"]) + details = [d for t in tasks for p in t["participants"] for d in p.get("code_url_details", [])] + live = sum(1 for d in details if d["status"] == "200") + dead = sum(1 for d in details if d["status"] in {"404", "unreachable"}) + unverified = sum(1 for d in details if d["status"] == "429") + evidenced = sum(1 for d in details if d["availability_evidence"]) + third_party = sum(len(p.get("third_party_urls", [])) for t in tasks for p in t["participants"]) + + lines += [ + "", + "## Coverage", + "", + f"- Total participant/notebook papers: {total_participants}", + f"- Tasks with a known coverage_ratio: {len(ratios)}/{len(tasks)}", + f"- Mean coverage_ratio (where known): {round(sum(ratios) / len(ratios), 3) if ratios else 'n/a'}", + f"- Umbrella overviews (one overview serving several sub-tasks): {umbrella}/{len(tasks)}", + f"- Participants with an extracted team_name: {named_teams}/{total_participants}", + "", + "`coverage_ratio` is null where the overview's claimed team count could not be", + "extracted; those entries sort last in the ranking rather than being scored.", + "", + "## Code links", + "", + f"- Participant papers with at least one resolved code/TIRA link: {with_code}/{total_participants}", + f"- Participant papers with no resolved link: {unresolved}/{total_participants}", + "", + f"- Code URLs stored: {len(details)} (live 200: {live}, dead 404/unreachable: {dead}, unverified 429 rate-limited: {unverified})", + f"- Of those, backed by an explicit code-availability statement: {evidenced}", + f"- Third-party dependency URLs excluded from code_urls: {third_party}", + "", + "Links are not filtered by HTTP status: a dead repository is still evidence the", + "team published code. Use `code_url_details[].status` to distinguish, and", + "`availability_evidence` to prefer links the authors explicitly released.", + "Dependencies the team merely used (transformers, nltk, pretrained weights) are", + "kept out of `code_urls` and listed per participant in `third_party_urls`.", + ] + + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Assemble final shared_tasks.jsonl / .csv / report.md from high-confidence candidate tasks.") + parser.add_argument("--confidence", type=str, default="high", choices=["high", "medium", "all"], help="Which candidate tasks to include (default: high only).") + parser.add_argument("--seed", type=int, default=42, help="Random seed for the spot-check sample.") + parser.add_argument("--target", type=int, default=50, help="Maximum corpus size to emit (PLAN.md target: 30-50).") + args = parser.parse_args() + + log_path = setup_logging() + logger = logging.getLogger("build_corpus") + logger.info("logging to %s", log_path) + random.seed(args.seed) + + if not CANDIDATES_PATH.exists(): + logger.error("missing %s — run group_tasks.py first", CANDIDATES_PATH) + sys.exit(1) + + tasks = [json.loads(line) for line in CANDIDATES_PATH.read_text(encoding="utf-8").splitlines()] + if args.confidence != "all": + tasks = [t for t in tasks if t["provenance"]["confidence"] == args.confidence] + logger.info("assembling corpus from %d tasks (confidence=%s)", len(tasks), args.confidence) + + fulltext_manifest = load_fulltext_manifest(logger) + for task in tasks: + join_counts(task, logger) + join_code_links(task, logger) + join_fulltext_paths(task, fulltext_manifest, logger) + + tasks = select_corpus(tasks, args.target, logger) + + if not validate(tasks, logger): + logger.error("validation failed — see errors above. Deliverables NOT written.") + sys.exit(1) + + FINAL_DIR.mkdir(parents=True, exist_ok=True) + write_jsonl(tasks, FINAL_DIR / "shared_tasks.jsonl") + write_csv(tasks, FINAL_DIR / "shared_tasks.csv") + write_report(tasks, FINAL_DIR / "report.md") + logger.info("wrote %d tasks to data/final/shared_tasks.jsonl, .csv, report.md", len(tasks)) + + +if __name__ == "__main__": + main() diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/extract_counts.py b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/extract_counts.py new file mode 100755 index 0000000..06a430d --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/extract_counts.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python +"""Stage 4 — extract claimed team/run counts from overview PDFs, scoped to the 44 +high-confidence task groups from Stage 3 (see PLAN.md section 3, Stage 4).""" +import argparse +import json +import logging +import re +import subprocess +import sys +from datetime import datetime +from pathlib import Path + +import requests + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +CANDIDATES_PATH = PROJECT_ROOT / "data" / "intermediate" / "all_candidates.jsonl" +PDF_RAW_DIR = PROJECT_ROOT / "data" / "raw" / "pdf" +COUNTS_DIR = PROJECT_ROOT / "data" / "intermediate" / "counts" +LOGS_DIR = PROJECT_ROOT / "logs" + +REQUEST_TIMEOUT_SECONDS = 30 + +# Restricting to the first ~15000 characters of extracted text (roughly the abstract, +# introduction, and any early results-summary paragraph for a typical CEUR paper) avoids +# picking up unrelated numbers from tables, results sections, or the bibliography further +# into longer overview papers. +SEARCH_WINDOW_CHARS = 15000 + +# Ordered by specificity: patterns naming *actual* participation ("received results +# from", "participating") are tried before a bare "N teams", because overview papers +# often report both a registration count and a (smaller) participation count in the same +# sentence — e.g. "98 teams registered ... we received results from 20 teams" — and only +# the latter is the correct denominator for coverage_ratio. +TEAM_PATTERNS = [ + re.compile(r"(\d+)\s+participating\s+teams", re.IGNORECASE), + re.compile(r"received\s+(?:results|submissions)\s+(?:coming\s+)?from\s+(\d+)\s+(?:distinct\s+|different\s+)?teams", re.IGNORECASE), + re.compile(r"(\d+)\s+distinct\s+teams", re.IGNORECASE), + re.compile(r"(\d+)\s+participating\s+groups", re.IGNORECASE), + re.compile(r"(\d+)\s+teams\s+(?:actively\s+)?participat\w*", re.IGNORECASE), + re.compile(r"(\d+)\s+teams\s+submitted", re.IGNORECASE), + re.compile(r"total\s+of\s+(\d+)\s+teams", re.IGNORECASE), + re.compile(r"(\d+)\s+teams\b(?!\s*(?:registered|up\b))", re.IGNORECASE), +] + +# Team/run counts for a CLEF shared task are never in the hundreds-of-thousands, let +# alone four digits shaped like a year. Real bug found on CENTRE@CLEF 2019: "CENTRE@CLEF +# 2019 teams up with the Open-Source IR Replicability Challenge" used "teams" as a verb +# ("teams up with"), and the generic fallback pattern grabbed the adjacent year 2019 as +# if it were a team count. A negative lookahead for "up" fixes that specific phrasing, +# but a plausibility bound catches this whole class of error regardless of wording. +MAX_PLAUSIBLE_COUNT = 999 + + +def is_plausible_count(n: int) -> bool: + return 0 < n <= MAX_PLAUSIBLE_COUNT +# "(? Path: + LOGS_DIR.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_path = LOGS_DIR / f"extract_counts_{timestamp}.log" + + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") + + file_handler = logging.FileHandler(log_path, mode="w") + file_handler.setFormatter(formatter) + root_logger.addHandler(file_handler) + + stream_handler = logging.StreamHandler(sys.stderr) + stream_handler.setFormatter(formatter) + root_logger.addHandler(stream_handler) + + return log_path + + +def fetch_pdf(url: str, dest_path: Path, logger: logging.Logger) -> bool: + if dest_path.exists(): + logger.info("cache hit: %s -> %s", url, dest_path) + return True + + dest_path.parent.mkdir(parents=True, exist_ok=True) + try: + response = requests.get(url, timeout=REQUEST_TIMEOUT_SECONDS, headers={"User-Agent": "uniagent-corpus-builder/0.1"}) + except requests.RequestException as exc: + logger.error("fetch failed: %s (%s)", url, exc) + return False + + logger.info("fetched: %s status=%d -> %s", url, response.status_code, dest_path) + if response.status_code != 200: + logger.warning("non-200 status for %s: %d", url, response.status_code) + return False + + dest_path.write_bytes(response.content) + return True + + +def parse_pdf_text(pdf_path: Path, txt_path: Path, logger: logging.Logger) -> str | None: + if txt_path.exists(): + logger.info("parse cache hit: %s", txt_path) + return txt_path.read_text(encoding="utf-8") + + result = subprocess.run( + ["lit", "parse", str(pdf_path), "--format", "text", "--no-ocr", "-o", str(txt_path)], + capture_output=True, text=True, + ) + if result.returncode != 0: + logger.error("lit parse failed for %s: %s", pdf_path, result.stderr.strip()) + return None + logger.info("parsed: %s -> %s", pdf_path, txt_path) + return txt_path.read_text(encoding="utf-8") + + +NUMBERED_HEADING_RE = re.compile(r"\n(\d+)\.\s+\S") + + +def abstract_and_intro_window(text: str) -> str: + """Restrict extraction to the abstract + introduction, ending at the start of the + first numbered *task* subsection (heading number >= 2 — typically "2. Task 1: ..."). + Real bug found on eRisk 2023: without this boundary, a per-task participation + sentence buried inside the Task 1 subsection ("37 runs from 10 participating teams") + outscored the actual lab-wide total in the introduction ("received results from 20 + teams") because it matched a higher-priority pattern. + + A second real bug found on eRisk 2025: this cannot simply take the *second* numbered + heading found and assume the first is "1. Introduction" — that paper has no numbered + "1." heading at all, so its first numbered heading is already "2. Task 1", and taking + the second heading overall ("3. Task 2") let the entire wrong Task-1-only subsection + leak into the window. Must explicitly look for a heading number >= 2.""" + for match in NUMBERED_HEADING_RE.finditer(text): + if int(match.group(1)) >= 2: + return text[:match.start()] + return text[:SEARCH_WINDOW_CHARS] + + +# Matches both "N runs for Task 1" and "N runs for the pilot task" — real case found on +# eRisk 2025: "67 runs for Task 1, 50 runs for Task 2, and 11 runs for the pilot task" +# only summed to 117 (missing the non-numbered "pilot task" fragment) against a true +# total of 128, because the original pattern required a task *number* on every fragment. +RUNS_PER_TASK_RE = re.compile(r"(\d+)\s+runs\s+for\s+(?:the\s+)?(?:task\s*\d+|\w+\s+task)", re.IGNORECASE) + + +def extract_run_count(window: str) -> int | None: + """Some overviews report runs as an un-totaled per-task breakdown in the intro + ("37 runs for Task 1, 48 runs for Task 2, and 20 runs for Task 3") rather than a + single total. Sum that breakdown when found — a real case (eRisk 2023) otherwise + silently yielded the first task's count (37) instead of the lab-wide total (105).""" + per_task_matches = [int(n) for n in RUNS_PER_TASK_RE.findall(window)] + plausible_per_task = [n for n in per_task_matches if is_plausible_count(n)] + if len(plausible_per_task) >= 2: + return sum(plausible_per_task) + return extract_count(window, RUN_PATTERNS) + + +def extract_count(window: str, patterns: list[re.Pattern]) -> int | None: + for pattern in patterns: + for match in pattern.finditer(window): + candidate = int(match.group(1)) + if is_plausible_count(candidate): + return candidate + return None + + +def process_task(task: dict, logger: logging.Logger) -> dict: + task_id = task["task_id"] + out_path = COUNTS_DIR / f"{task_id}.json" + if out_path.exists(): + logger.info("counts cache hit: %s", out_path) + return json.loads(out_path.read_text(encoding="utf-8")) + + pdf_dir = PDF_RAW_DIR / task_id + pdf_path = pdf_dir / "overview.pdf" + txt_path = pdf_dir / "overview.txt" + + if not fetch_pdf(task["overview"]["pdf_url"], pdf_path, logger): + result = {"teams": None, "runs": None} + else: + text = parse_pdf_text(pdf_path, txt_path, logger) + if text is None: + result = {"teams": None, "runs": None} + else: + window = abstract_and_intro_window(text) + teams = extract_count(window, TEAM_PATTERNS) + runs = extract_run_count(window) + + # PLAN.md's own coverage_ratio bound doubles as a sanity check on the + # extraction: more notebook papers than claimed teams (ratio > 1.5) means the + # regex almost certainly grabbed the wrong number, e.g. a real case (LongEval + # 2023) where "14 and 4 teams participated in Task 1 and Task 2, respectively" + # yielded 4 (only the second, elliptically-written figure) against 14 actual + # notebook papers. Null it out rather than keep a number known to be wrong. + notebook_papers = len(task["participants"]) + if teams is not None and notebook_papers / teams > 1.5: + logger.warning( + "%s: claimed teams=%d implausible against %d notebook papers (ratio %.2f > 1.5) — discarding as mis-parsed", + task_id, teams, notebook_papers, notebook_papers / teams, + ) + teams = None + + result = {"teams": teams, "runs": runs} + if teams is None: + logger.warning("%s: no team-count pattern matched — storing null", task_id) + if runs is None: + logger.warning("%s: no run-count pattern matched — storing null", task_id) + + COUNTS_DIR.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(result, indent=2), encoding="utf-8") + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description="Extract claimed team/run counts from overview PDFs.") + parser.add_argument("--confidence", type=str, default="high", choices=["high", "medium", "all"], help="Which candidate tasks to process (default: high only).") + args = parser.parse_args() + + log_path = setup_logging() + logger = logging.getLogger("extract_counts") + logger.info("logging to %s", log_path) + + if not CANDIDATES_PATH.exists(): + logger.error("missing %s — run group_tasks.py first", CANDIDATES_PATH) + sys.exit(1) + + tasks = [json.loads(line) for line in CANDIDATES_PATH.read_text(encoding="utf-8").splitlines()] + if args.confidence != "all": + tasks = [t for t in tasks if t["provenance"]["confidence"] == args.confidence] + + matched_teams = matched_runs = 0 + for task in tasks: + result = process_task(task, logger) + if result["teams"] is not None: + matched_teams += 1 + if result["runs"] is not None: + matched_runs += 1 + + logger.info( + "processed %d tasks: team counts found for %d, run counts found for %d", + len(tasks), matched_teams, matched_runs, + ) + + +if __name__ == "__main__": + main() diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/fetch_volumes.py b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/fetch_volumes.py new file mode 100755 index 0000000..e9bdb8a --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/fetch_volumes.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +"""Stage 1 — fetch and cache CEUR-WS volume index pages and DBLP working-notes records.""" +import argparse +import logging +import sys +import time +from datetime import datetime +from pathlib import Path + +import requests + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +CEUR_RAW_DIR = PROJECT_ROOT / "data" / "raw" / "ceur" +DBLP_RAW_DIR = PROJECT_ROOT / "data" / "raw" / "dblp" +LOGS_DIR = PROJECT_ROOT / "logs" + +CEUR_URL_TEMPLATE = "https://ceur-ws.org/Vol-{volume}/" +REQUEST_TIMEOUT_SECONDS = 30 +REQUEST_DELAY_SECONDS = 1.0 + +# Parent venue | year | CEUR-WS volume | DBLP working-notes record (PLAN.md section 2) +VOLUME_MAP = [ + {"parent_venue": "CLEF", "year": 2025, "volume": "4038", "dblp_url": "https://dblp.org/db/conf/clef/clef2025w.html"}, + {"parent_venue": "CLEF", "year": 2024, "volume": "3740", "dblp_url": "https://dblp.org/db/conf/clef/clef2024w.html"}, + {"parent_venue": "CLEF", "year": 2023, "volume": "3497", "dblp_url": "https://dblp.org/db/conf/clef/clef2023w.html"}, + {"parent_venue": "CLEF", "year": 2022, "volume": "3180", "dblp_url": "https://dblp.org/db/conf/clef/clef2022w.html"}, + {"parent_venue": "CLEF", "year": 2021, "volume": "2936", "dblp_url": "https://dblp.org/db/conf/clef/clef2021w.html"}, + {"parent_venue": "CLEF", "year": 2020, "volume": "2696", "dblp_url": "https://dblp.org/db/conf/clef/clef2020w.html"}, + {"parent_venue": "CLEF", "year": 2019, "volume": "2380", "dblp_url": "https://dblp.org/db/conf/clef/clef2019w.html"}, + {"parent_venue": "CLEF", "year": 2018, "volume": "2125", "dblp_url": "https://dblp.org/db/conf/clef/clef2018w.html"}, +] + + +def setup_logging() -> Path: + LOGS_DIR.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_path = LOGS_DIR / f"fetch_volumes_{timestamp}.log" + + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") + + file_handler = logging.FileHandler(log_path, mode="w") + file_handler.setFormatter(formatter) + root_logger.addHandler(file_handler) + + stream_handler = logging.StreamHandler(sys.stderr) + stream_handler.setFormatter(formatter) + root_logger.addHandler(stream_handler) + + return log_path + + +def fetch_url(url: str, dest_path: Path, logger: logging.Logger) -> str: + """Fetch url to dest_path unless already cached. Returns "cached", "fetched", or "failed".""" + if dest_path.exists(): + logger.info("cache hit: %s -> %s", url, dest_path) + return "cached" + + dest_path.parent.mkdir(parents=True, exist_ok=True) + try: + response = requests.get(url, timeout=REQUEST_TIMEOUT_SECONDS, headers={"User-Agent": "uniagent-corpus-builder/0.1"}) + except requests.RequestException as exc: + logger.error("fetch failed: %s (%s)", url, exc) + return "failed" + + logger.info("fetched: %s status=%d -> %s", url, response.status_code, dest_path) + if response.status_code != 200: + logger.warning("non-200 status for %s: %d", url, response.status_code) + return "failed" + + dest_path.write_text(response.text, encoding="utf-8") + return "fetched" + + +def fetch_all(volumes: list[dict], logger: logging.Logger) -> None: + for entry in volumes: + ceur_url = CEUR_URL_TEMPLATE.format(volume=entry["volume"]) + ceur_dest = CEUR_RAW_DIR / f"Vol-{entry['volume']}.html" + ceur_status = fetch_url(ceur_url, ceur_dest, logger) + if ceur_status == "fetched": + time.sleep(REQUEST_DELAY_SECONDS) + if ceur_status == "failed": + logger.warning("skipping %s %s due to CEUR fetch failure", entry["parent_venue"], entry["year"]) + + dblp_dest = DBLP_RAW_DIR / f"{entry['parent_venue']}{entry['year']}.html" + dblp_status = fetch_url(entry["dblp_url"], dblp_dest, logger) + if dblp_status == "fetched": + time.sleep(REQUEST_DELAY_SECONDS) + if dblp_status == "failed": + logger.warning( + "DBLP cross-check source unavailable for %s %s (%s) — Stage 2 will log unmatched papers, not fail", + entry["parent_venue"], entry["year"], entry["dblp_url"], + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Fetch and cache CEUR-WS volume index pages and DBLP working-notes records.") + parser.add_argument("--volume", type=str, default=None, help="Fetch only this CEUR-WS volume number (e.g. 3497). Default: fetch all volumes in the map.") + args = parser.parse_args() + + log_path = setup_logging() + logger = logging.getLogger("fetch_volumes") + logger.info("logging to %s", log_path) + + volumes = VOLUME_MAP + if args.volume is not None: + volumes = [entry for entry in VOLUME_MAP if entry["volume"] == args.volume] + if not volumes: + logger.error("volume %s not found in VOLUME_MAP", args.volume) + sys.exit(1) + + fetch_all(volumes, logger) + + +if __name__ == "__main__": + main() diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/find_code.py b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/find_code.py new file mode 100755 index 0000000..860eba0 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/find_code.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python +"""Stage 5 — resolve participant code links, scoped to the 44 high-confidence task +groups from Stage 3 (see PLAN.md section 3, Stage 5).""" +import argparse +import json +import logging +import re +import subprocess +import sys +from datetime import datetime +from pathlib import Path +from urllib.parse import urlparse + +import requests + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +CANDIDATES_PATH = PROJECT_ROOT / "data" / "intermediate" / "all_candidates.jsonl" +PDF_RAW_DIR = PROJECT_ROOT / "data" / "raw" / "pdf" +CODE_DIR = PROJECT_ROOT / "data" / "intermediate" / "code" +LOGS_DIR = PROJECT_ROOT / "logs" + +REQUEST_TIMEOUT_SECONDS = 30 +HEAD_TIMEOUT_SECONDS = 10 + +CODE_URL_RE = re.compile( + r"https?://(?:www\.)?(?:github\.com|gitlab\.[a-z0-9.\-]+|zenodo\.org/record|huggingface\.co)/\S+", + re.IGNORECASE, +) +TIRA_URL_RE = re.compile(r"https?://(?:www\.)?tira\.io/\S+", re.IGNORECASE) +TIRA_DOCKER_RE = re.compile(r"\bdocker\.io/[\w\-./:]+", re.IGNORECASE) + +# Trailing punctuation that's part of the surrounding sentence, not the URL itself +# (e.g. "see https://github.com/x/y." at the end of a sentence). +TRAILING_PUNCTUATION_RE = re.compile(r"[.,;:)\]]+$") + +# A paper's bibliography cites the tools it used; those are not the team's own code. +# PLAN.md Stage 5 says code links live in a footnote or a "Reproducibility"/"Availability" +# section, so everything from the reference list onward is dropped before matching. +REFERENCES_HEADING_RE = re.compile(r"(?im)^[ \t]*(?:\d+\.?\s*)?(references|bibliography|works cited)[ \t]*$") + +# Namespaces that publish the field's shared infrastructure and pretrained weights. A URL +# under one of these is a dependency the team *used*, never the code the team *wrote*. +# Audit found these made up ~48% of stored code_urls, so the field misrepresented +# ubiquitous libraries (transformers, nltk, keras, trec_eval) as team submissions. +THIRD_PARTY_NAMESPACES = { + "huggingface", "pytorch", "tensorflow", "keras", "fchollet", "scikit-learn", "scipy", + "numpy", "pandas-dev", "explosion", "nltk", "facebookresearch", "facebookai", "facebook", + "meta-llama", "google", "google-research", "google-bert", "googlecreativelab", "microsoft", + "openai", "allenai", "ukplab", "sentence-transformers", "cardiffnlp", "castorini", + "usnistgov", "fasterxml", "rare-technologies", "stanfordnlp", "flairnlp", "deepset-ai", + "deepset", "mistralai", "qwen", "bigscience", "eleutherai", "tiiuae", "intfloat", "baai", + "sentencepiece", "apache", "elastic", "terrier-org", "terrierteam", "explosion-ai", + "spacy-io", "dmlc", "xgboost", "unslothai", "langchain-ai", "jina-ai", "nomic-ai", + "salesforce", "databricks", "mosaicml", "togethercomputer", "thudm", "internlm", + "openai-community", "datasets", "sebastianruder", "zenodo", +} + +# Hugging Face paths are frequently a bare pretrained-model name with no org segment +# (e.g. huggingface.co/bert-base-uncased) — also a dependency, not team code. +BARE_MODEL_NAME_RE = re.compile( + r"^(bert|roberta|distilbert|albert|xlm|xlnet|gpt2|gpt-2|t5|flan|deberta|electra|bart|mbart|mt5|opt|bloom|llama)[\w.\-]*$", + re.IGNORECASE, +) + +# Phrases that mark a link as the authors' own released artifact rather than a passing +# mention. Recorded as evidence so a consumer can prefer high-signal links. +CODE_AVAILABILITY_RE = re.compile( + r"(our code|source code|code is available|code are available|code can be found|is available at|are available at|" + r"we release|publicly available|made available|reproducib|our implementation|our repository|" + r"github repository|our system is available|we provide the code|implementation is available)", + re.IGNORECASE, +) +OWNER_RE = re.compile(r"^https?://(?:www\.)?(?:github\.com|gitlab\.[^/]+|huggingface\.co|zenodo\.org)/([^/\s?#]+)", re.IGNORECASE) + + +def strip_bibliography(text: str) -> str: + """Drop the reference list. Uses the last heading found, and only when it sits past + the first third of the document, so an in-body mention of the word "references" + does not truncate the paper.""" + matches = list(REFERENCES_HEADING_RE.finditer(text)) + if not matches: + return text + last = matches[-1] + if last.start() < len(text) * 0.3: + return text + return text[: last.start()] + + +def is_third_party(url: str) -> bool: + match = OWNER_RE.match(url) + if not match: + return False + owner = match.group(1).lower() + return owner in THIRD_PARTY_NAMESPACES or bool(BARE_MODEL_NAME_RE.match(owner)) + + +def has_availability_evidence(text: str, position: int) -> bool: + """Look at the sentence-ish window around the URL for a code-release phrase.""" + window = text[max(0, position - 240) : position + 120] + return bool(CODE_AVAILABILITY_RE.search(window)) + + +def setup_logging() -> Path: + LOGS_DIR.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_path = LOGS_DIR / f"find_code_{timestamp}.log" + + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") + + file_handler = logging.FileHandler(log_path, mode="w") + file_handler.setFormatter(formatter) + root_logger.addHandler(file_handler) + + stream_handler = logging.StreamHandler(sys.stderr) + stream_handler.setFormatter(formatter) + root_logger.addHandler(stream_handler) + + return log_path + + +def clean_url(url: str) -> str: + return TRAILING_PUNCTUATION_RE.sub("", url) + + +def extract_links(text: str) -> tuple[list[dict], list[str], list[str]]: + """Return (candidate_code_links, third_party_urls, tira_refs) from a paper's text. + + Candidates are the links plausibly pointing at the team's own artifact: found outside + the bibliography and not under a known third-party namespace. Each carries an + `evidence` flag recording whether it appeared in a code-availability context. + Third-party URLs are returned separately rather than discarded, so the exclusion stays + auditable instead of silently dropping data.""" + body = strip_bibliography(text) + + candidates: dict[str, bool] = {} + third_party: set[str] = set() + for match in CODE_URL_RE.finditer(body): + url = clean_url(match.group(0)) + if not url: + continue + if is_third_party(url): + third_party.add(url) + continue + evidence = has_availability_evidence(body, match.start()) + candidates[url] = candidates.get(url, False) or evidence + + tira_refs = sorted( + {clean_url(u) for u in TIRA_URL_RE.findall(body)} + | {clean_url(u) for u in TIRA_DOCKER_RE.findall(body)} + ) + candidate_links = [{"url": u, "evidence": candidates[u]} for u in sorted(candidates)] + return candidate_links, sorted(third_party), tira_refs + + +def validate_url(url: str, logger: logging.Logger) -> str: + """HEAD-check a code URL. Returns a status label; a dead link is kept (not dropped) + since per PLAN.md it's still evidence the team had a repo.""" + try: + response = requests.head(url, timeout=HEAD_TIMEOUT_SECONDS, allow_redirects=True, headers={"User-Agent": "uniagent-corpus-builder/0.1"}) + status = str(response.status_code) + except requests.RequestException as exc: + status = "unreachable" + logger.warning("HEAD check failed for %s: %s", url, exc) + logger.info("code URL check: %s -> %s", url, status) + return status + + +def fetch_pdf(url: str, dest_path: Path, logger: logging.Logger) -> bool: + if dest_path.exists(): + logger.info("cache hit: %s -> %s", url, dest_path) + return True + + dest_path.parent.mkdir(parents=True, exist_ok=True) + try: + response = requests.get(url, timeout=REQUEST_TIMEOUT_SECONDS, headers={"User-Agent": "uniagent-corpus-builder/0.1"}) + except requests.RequestException as exc: + logger.error("fetch failed: %s (%s)", url, exc) + return False + + logger.info("fetched: %s status=%d -> %s", url, response.status_code, dest_path) + if response.status_code != 200: + logger.warning("non-200 status for %s: %d", url, response.status_code) + return False + + dest_path.write_bytes(response.content) + return True + + +def parse_pdf_text(pdf_path: Path, txt_path: Path, logger: logging.Logger) -> str | None: + if txt_path.exists(): + logger.info("parse cache hit: %s", txt_path) + return txt_path.read_text(encoding="utf-8") + + result = subprocess.run( + ["lit", "parse", str(pdf_path), "--format", "text", "--no-ocr", "-o", str(txt_path)], + capture_output=True, text=True, + ) + if result.returncode != 0: + logger.error("lit parse failed for %s: %s", pdf_path, result.stderr.strip()) + return None + logger.info("parsed: %s -> %s", pdf_path, txt_path) + return txt_path.read_text(encoding="utf-8") + + +def pdf_filename_for(pdf_url: str) -> str: + stem = Path(urlparse(pdf_url).path).stem + return re.sub(r"[^a-zA-Z0-9_\-]", "_", stem) + + +def process_participant(task_id: str, participant: dict, logger: logging.Logger) -> dict: + pdf_url = participant["pdf_url"] + stem = pdf_filename_for(pdf_url) + pdf_dir = PDF_RAW_DIR / task_id + pdf_path = pdf_dir / f"{stem}.pdf" + txt_path = pdf_dir / f"{stem}.txt" + + empty = {"pdf_url": pdf_url, "code_urls": [], "third_party_urls": [], "tira_refs": []} + if not fetch_pdf(pdf_url, pdf_path, logger): + return empty + + text = parse_pdf_text(pdf_path, txt_path, logger) + if text is None: + return empty + + candidates, third_party, tira_refs = extract_links(text) + if third_party: + logger.info("%s: excluded %d third-party dependency link(s)", pdf_url, len(third_party)) + if not candidates and not tira_refs: + logger.info("%s: no code links found", pdf_url) + return {"pdf_url": pdf_url, "code_urls": [], "third_party_urls": third_party, "tira_refs": tira_refs} + + validated = [ + {"url": c["url"], "status": validate_url(c["url"], logger), "evidence": c["evidence"]} + for c in candidates + ] + return {"pdf_url": pdf_url, "code_urls": validated, "third_party_urls": third_party, "tira_refs": tira_refs} + + +def process_task(task: dict, logger: logging.Logger) -> list[dict]: + task_id = task["task_id"] + out_path = CODE_DIR / f"{task_id}.json" + if out_path.exists(): + logger.info("code cache hit: %s", out_path) + return json.loads(out_path.read_text(encoding="utf-8")) + + results = [process_participant(task_id, p, logger) for p in task["participants"]] + + CODE_DIR.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(results, indent=2, ensure_ascii=False), encoding="utf-8") + return results + + +def main() -> None: + parser = argparse.ArgumentParser(description="Resolve participant code links for candidate tasks.") + parser.add_argument("--confidence", type=str, default="high", choices=["high", "medium", "all"], help="Which candidate tasks to process (default: high only).") + parser.add_argument("--task-id", type=str, default=None, help="Process only this single task_id (for testing).") + args = parser.parse_args() + + log_path = setup_logging() + logger = logging.getLogger("find_code") + logger.info("logging to %s", log_path) + + if not CANDIDATES_PATH.exists(): + logger.error("missing %s — run group_tasks.py first", CANDIDATES_PATH) + sys.exit(1) + + tasks = [json.loads(line) for line in CANDIDATES_PATH.read_text(encoding="utf-8").splitlines()] + if args.task_id is not None: + tasks = [t for t in tasks if t["task_id"] == args.task_id] + elif args.confidence != "all": + tasks = [t for t in tasks if t["provenance"]["confidence"] == args.confidence] + + total_participants = 0 + with_code = 0 + for task in tasks: + results = process_task(task, logger) + total_participants += len(results) + with_code += sum(1 for r in results if r["code_urls"] or r["tira_refs"]) + logger.info("%s: %d/%d participants have a resolved link", task["task_id"], sum(1 for r in results if r["code_urls"] or r["tira_refs"]), len(results)) + + logger.info("processed %d tasks, %d participant papers: %d have at least one resolved code/TIRA link", len(tasks), total_participants, with_code) + + +if __name__ == "__main__": + main() diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/group_tasks.py b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/group_tasks.py new file mode 100755 index 0000000..7906b90 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/group_tasks.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python +"""Stage 3 — group each parsed section's papers into shared-task entries, linking one +overview paper to its participant/notebook papers. See PLAN.md section 3. + +IMPORTANT DEVIATION FROM PLAN.MD, discovered while building this stage: PLAN.md's rule 1 +("section grouping") assumes each overview paper is immediately followed by its own +participants until the next overview. Real CEUR-WS volumes instead front-load ALL of a +lab's overview papers first, then list participants in an order that is neither strictly +per-task nor alphabetical (verified across all 8 CLEF volumes). So for any section with +2+ overviews, title-keyword matching (PLAN.md's rule 3, "title_heuristic") is the primary +assignment mechanism here, not a rare last resort — and those groups are always written to +needs_review.jsonl with confidence="medium", never auto-promoted to the main corpus. +Sections with exactly one overview are unaffected: positional grouping is trivially +correct there and stays confidence="high". +""" +import argparse +import json +import logging +import re +import sys +from datetime import date, datetime +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +SECTIONS_DIR = PROJECT_ROOT / "data" / "intermediate" / "sections" +INTERMEDIATE_DIR = PROJECT_ROOT / "data" / "intermediate" +LOGS_DIR = PROJECT_ROOT / "logs" + +OVERVIEW_TITLE_RE = re.compile(r"\boverview\b|\bextended abstract\b", re.IGNORECASE) + +# Organizer-written task descriptions that never use the word "overview". Real bug found +# in audit: the ELOQUENT 2024 section publishes three organizer task papers, but only +# "Overview of the CLEF-2024 Eloquent Lab: Task 2 on HalluciGen" says "overview" — so +# "ELOQUENT 2024 — Topical Quiz Task" and "ELOQUENT 2024 — Robustness Task" were filed as +# participant submissions, contaminating one task and losing two others entirely. +# The distinguishing shape is lab-branding + year + a dash + a task name, with no team +# attribution. Verified across all 8 volumes: adds exactly those 2 real detections and +# no false positives. +# +# An author-overlap rule (paper sharing authors with a detected overview) was evaluated +# for this and REJECTED: lab organizers routinely also submit competing systems ("DPRL +# Systems in the CLEF 2021 ARQMath Lab", "Organiser Team at ImageCLEFlifelog 2020"), so +# it flagged ~48 genuine participant papers. +ORGANIZER_TITLE_RE = re.compile(r"^\S+\s+20\d\d\s*[—–-]\s*.*\btask\b\s*$", re.IGNORECASE) +BEST_OF_LABS_RE = re.compile(r"\bbest of (the )?labs?\b", re.IGNORECASE) +STOPWORDS = { + "a", "an", "the", "of", "in", "on", "at", "for", "to", "and", "or", "with", + "overview", "task", "tasks", "lab", "labs", "extended", "abstract", "clef", + # Generic shared-task/ML vocabulary. Real bug found reviewing Vol-2936 PAN: a "Hate + # Speech Spreader Detection" paper (whose actual overview isn't published in this + # volume at all) got routed into "Style Change Detection" purely because "detection" + # was the only word it shared with any sibling overview, and — being section-locally + # unique by chance — was scored as fully discriminative. These words describe the + # shared-task *format*, not any specific task's topic, and recur across nearly every + # CLEF lab regardless of subject, so they carry no assignment signal within a section. + "detection", "classification", "identification", "prediction", "recognition", + "analysis", "approach", "using", "based", "model", "models", "system", "systems", + "method", "methods", "challenge", "evaluation", "text", +} + + +def setup_logging() -> Path: + LOGS_DIR.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_path = LOGS_DIR / f"group_tasks_{timestamp}.log" + + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") + + file_handler = logging.FileHandler(log_path, mode="w") + file_handler.setFormatter(formatter) + root_logger.addHandler(file_handler) + + stream_handler = logging.StreamHandler(sys.stderr) + stream_handler.setFormatter(formatter) + root_logger.addHandler(stream_handler) + + return log_path + + +def tokenize(title: str) -> set[str]: + # Split camelCase compounds (e.g. "BirdCLEF", "GeoLifeCLEF", "MedProcNER") before + # lowercasing — otherwise "BirdCLEF" and "GeoLifeCLEF" collide as opaque single + # tokens and only their shared, non-discriminative "CLEF" suffix would ever be + # compared. "CLEF" itself still gets filtered out afterward via STOPWORDS. + spaced = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", " ", title) + words = re.findall(r"[a-z0-9]+", spaced.lower()) + return {w for w in words if w not in STOPWORDS and len(w) > 1} + + +def slugify(text: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") + return re.sub(r"-{2,}", "-", slug) + + +def extract_venue(lab_name: str) -> str: + """Best-effort extraction of a short venue/lab acronym from a full section heading, + e.g. "Overview of ... (BioASQ)" -> "BioASQ", "BioASQ: Large-scale ..." -> "BioASQ", + "PAN Lab on Digital Text Forensics" -> "PAN". Falls back to the full heading when no + recognizable pattern applies — imperfect but non-blocking; venue is a display field.""" + paren_match = re.search(r"\(([^()]+)\)\s*$", lab_name) + if paren_match: + return paren_match.group(1).strip() + + colon_match = re.match(r"^([^:]+):", lab_name) + if colon_match: + candidate = colon_match.group(1).strip() + return re.sub(r"\s+Lab$", "", candidate, flags=re.IGNORECASE) + + dash_match = re.match(r"^([^-]+)-", lab_name) + if dash_match: + candidate = dash_match.group(1).strip() + if candidate and " " not in candidate.strip("@"): + return candidate + + first_word_match = re.match(r"^(\S+)\s+Lab\b", lab_name) + if first_word_match: + return first_word_match.group(1) + + return lab_name + + +def clean_task_name(overview_title: str) -> str: + """Strip boilerplate ("Overview of the ...", trailing "at PAN 2023" / "in CLEF2023") + from an overview title to produce a human-readable task name. Best-effort — see + task_id, which only needs uniqueness, not cosmetic perfection.""" + title = overview_title.rstrip(".") + + marker = re.search(r"(overview|abstract) of (the )?", title, re.IGNORECASE) + if marker: + title = title[marker.end():] + + task_n = re.search(r"\bTask\s+\d+\s*[:\-]?\s*(on|of)?\s*", title, re.IGNORECASE) + if task_n: + remainder = title[task_n.end():].strip() + if remainder: + title = remainder + + title = re.sub(r"\s+(Task\s+)?(at|in)\s+.+$", "", title, flags=re.IGNORECASE) + title = title.strip().rstrip(":").strip() + return title or overview_title.rstrip(".") + + +def find_overview_indices(papers: list[dict], logger: logging.Logger, lab_name: str) -> list[int]: + indices = [ + i for i, p in enumerate(papers) + if OVERVIEW_TITLE_RE.search(p["title"]) or ORGANIZER_TITLE_RE.search(p["title"]) + ] + if not indices and papers: + logger.warning("no title matched overview keyword in section %r — using first paper as overview", lab_name) + indices = [0] + return indices + + +def split_best_of_labs(papers: list[dict], logger: logging.Logger) -> tuple[list[dict], list[dict]]: + kept, excluded = [], [] + for p in papers: + (excluded if BEST_OF_LABS_RE.search(p["title"]) else kept).append(p) + if excluded: + logger.info("excluded %d 'Best of Labs' re-publication paper(s)", len(excluded)) + return kept, excluded + + +def assign_by_position(papers: list[dict], overview_indices: list[int]) -> list[dict]: + """Single-overview case: everything else in the section is that task's participants.""" + overview_idx = overview_indices[0] + return [p for i, p in enumerate(papers) if i != overview_idx] + + +def assign_by_title_match(papers: list[dict], overview_indices: list[int], logger: logging.Logger, lab_name: str) -> dict[int, list[dict]]: + """Multi-overview case: assign each participant to the overview whose title shares + the most title-keyword weight with it. Each token is weighted 1/(number of overviews + in this section containing it) — a token unique to one overview counts fully, a token + shared by two related overviews (e.g. both GeoLifeCLEF and PlantCLEF mention "plant") + still counts partially instead of being discarded outright. A hard "must be + section-locally unique" cutoff was tried first and discarded real signal on any + genuinely related pair of overviews, letting an unrelated one-off token collision + (e.g. "Zero-Shot" vs "Few-Shot" both tokenizing to "shot") win by default.""" + overview_tokens = {i: tokenize(papers[i]["title"]) for i in overview_indices} + token_counts: dict[str, int] = {} + for tokens in overview_tokens.values(): + for tok in tokens: + token_counts[tok] = token_counts.get(tok, 0) + 1 + token_weight = {tok: 1.0 / count for tok, count in token_counts.items()} + + assignments: dict[int, list[dict]] = {i: [] for i in overview_indices} + overview_set = set(overview_indices) + unassigned = 0 + + tied = 0 + for i, paper in enumerate(papers): + if i in overview_set: + continue + paper_tokens = tokenize(paper["title"]) + scores = { + ov_i: sum(token_weight[tok] for tok in (paper_tokens & tokens)) + for ov_i, tokens in overview_tokens.items() + } + best_score = max(scores.values(), default=0) + if best_score == 0: + unassigned += 1 + continue + best_overviews = [ov_i for ov_i, score in scores.items() if score == best_score] + if len(best_overviews) > 1: + # A genuine tie is worse than a zero score — resolving it arbitrarily (e.g. + # via dict/max() iteration order) would silently invent an assignment with no + # actual evidence behind it. Drop it, same as an unmatched paper. + tied += 1 + continue + assignments[best_overviews[0]].append(paper) + + if unassigned: + logger.warning( + "section %r: %d/%d participant papers could not be confidently matched to any overview by title keywords — dropped", + lab_name, unassigned, len(papers) - len(overview_indices), + ) + if tied: + logger.warning( + "section %r: %d/%d participant papers tied between 2+ overviews by title keyword score — dropped rather than guessed", + lab_name, tied, len(papers) - len(overview_indices), + ) + return assignments + + +TASK_NUMBER_RE = re.compile(r"\btask\s*(\d+)\b", re.IGNORECASE) + + +def declared_task_number(title: str) -> str | None: + match = TASK_NUMBER_RE.search(title) + return match.group(1) if match else None + + +def has_hidden_second_task(overview_title: str, participants: list[dict]) -> bool: + """Real bug found reviewing CLEF eHealth 2021: a section had two real overview + papers (SpRadIE Task 1 and Consumer Health Search Task 2), but the CHS one is titled + "Consumer Health Search at CLEF eHealth 2021" — no "overview" or "extended abstract" + anywhere — so find_overview_indices() missed it, leaving the section looking + single-overview and "high confidence" when it wasn't. Positional grouping then dumped + every CHS participant (and the CHS overview itself) under the SpRadIE task. + + The generalizable signal: when the *lone detected* overview explicitly names its own + task number ("Overview of ... Task 1 - SpRadIE"), it is task-specific, not a lab-wide + umbrella — so a "participant" declaring a *different* task number is real evidence of + a second, undetected task, not just a subtask covered by this same overview. This does + NOT fire on umbrella overviews with no task number of their own (verified against 6 + other CLEF labs — ChEMU, Touché, eRisk, MC2, QuantumCLEF — where participants + routinely reference "Task 2" etc. as a subtask of one single all-encompassing + overview; none of those overview titles declare their own task number).""" + own_task = declared_task_number(overview_title) + if own_task is None: + return False + return any( + (p_task := declared_task_number(p["title"])) is not None and p_task != own_task + for p in participants + ) + + +MULTI_TASK_TITLE_RE = re.compile(r"\btasks\b", re.IGNORECASE) + +# "TeamName at Venue 2023: ..." / "TeamName@Venue: ..." — the two shapes CEUR working-note +# titles actually use for team attribution. Anything else yields null rather than a guess. +TEAM_AT_RE = re.compile(r"^\s*(?:team\s+)?(?P[^:]{2,40}?)\s+at\s+\S", re.IGNORECASE) +TEAM_HANDLE_RE = re.compile(r"^\s*(?:team\s+)?(?P[\w.\-]{2,30}@[\w.\-]{2,30})\s*[:,]", re.IGNORECASE) + + +def extract_team_name(title: str) -> str | None: + """Best-effort team name from a working-note title. Returns None when no clear + attribution pattern matches — never guesses (AGENT.md: write null, don't invent).""" + handle = TEAM_HANDLE_RE.match(title) + if handle: + return handle.group("team").strip() + at_match = TEAM_AT_RE.match(title) + if at_match: + team = at_match.group("team").strip().strip(",") + # "UNSL's participation at eRisk 2021" names the team UNSL, not "UNSL's + # participation" — drop the possessive tail so the field holds just the team. + team = re.sub(r"[’']s\s+(participation|submission|approach|system|contribution)$", "", team, flags=re.IGNORECASE).strip() + # Guard against sentence-like prefixes ("A Comparative Study of X at ...") which + # are descriptions, not team names. + if len(team.split()) <= 4 and not team.lower().startswith(("a ", "an ", "the ")): + return team + return None + + +def is_umbrella_overview(overview_title: str, participants: list[dict]) -> bool: + """PLAN.md section 1: is_umbrella is true when the only available overview for these + participants is a combined multi-task overview rather than a task-specific one. + + Detected either from the overview's own title naming multiple tasks ("BioASQ Tasks 11b + and Synergy11") or from its participants declaring two or more distinct task numbers, + which means one overview is serving several sub-tasks. Best-effort: a lab whose + overview and participant titles both omit task numbers entirely will read as + non-umbrella even if it ran several sub-tasks.""" + if MULTI_TASK_TITLE_RE.search(overview_title): + return True + task_numbers = {declared_task_number(p["title"]) for p in participants} + task_numbers.discard(None) + return len(task_numbers) >= 2 + + +def group_section(section: dict, entry: dict, logger: logging.Logger, extracted_at: str) -> list[dict]: + lab_name = section["lab_name"] + papers, best_of_labs = split_best_of_labs(section["papers"], logger) + if not papers: + return [] + + overview_indices = find_overview_indices(papers, logger, lab_name) + tasks = [] + + if len(overview_indices) == 1: + overview = papers[overview_indices[0]] + participants = assign_by_position(papers, overview_indices) + if has_hidden_second_task(overview["title"], participants): + logger.warning( + "section %r: overview %r declares its own task number but a participant declares a different one — " + "likely a second, undetected overview in this section; downgrading to confidence=medium for review", + lab_name, overview["title"], + ) + tasks.append(build_task_record(entry, lab_name, overview, participants, "section_grouping", "medium", extracted_at)) + else: + tasks.append(build_task_record(entry, lab_name, overview, participants, "section_grouping", "high", extracted_at)) + else: + assignments = assign_by_title_match(papers, overview_indices, logger, lab_name) + for ov_idx in overview_indices: + overview = papers[ov_idx] + participants = assignments[ov_idx] + tasks.append(build_task_record(entry, lab_name, overview, participants, "title_heuristic", "medium", extracted_at)) + + rejected = [t for t in tasks if not t["participants"]] + for t in rejected: + logger.warning("rejecting task %r: 0 participants (PLAN.md pitfall 4)", t["task_id"]) + return [t for t in tasks if t["participants"]] + + +def build_task_record(entry: dict, lab_name: str, overview: dict, participants: list[dict], method: str, confidence: str, extracted_at: str) -> dict: + venue = extract_venue(lab_name) + task_name = clean_task_name(overview["title"]) + task_id = f"{entry['parent_venue'].lower()}{entry['year']}-{slugify(venue)}-{slugify(task_name)}" + + return { + "task_id": task_id, + "venue": venue, + "parent_venue": entry["parent_venue"], + "year": entry["year"], + "task_name": task_name, + "ceur_volume": entry["volume"], + "overview": { + "title": overview["title"], + "pdf_url": overview["pdf_url"], + "authors": overview["authors"], + # Only a single-overview section can be serving as an umbrella; when a section + # has several task-specific overviews, each task has its own dedicated one. + "is_umbrella": method == "section_grouping" and is_umbrella_overview(overview["title"], participants), + }, + "participants": [ + { + "title": p["title"], + "authors": p["authors"], + "pdf_url": p["pdf_url"], + "team_name": extract_team_name(p["title"]), + "code_urls": [], + "tira_refs": [], + } + for p in participants + ], + "counts": { + "notebook_papers": len(participants), + "teams_claimed_in_overview": None, + "runs_claimed_in_overview": None, + "coverage_ratio": None, + }, + "provenance": { + "task_assignment_method": method, + "confidence": confidence, + "extracted_at": extracted_at, + }, + } + + +def validate(tasks: list[dict], logger: logging.Logger) -> None: + task_ids = [t["task_id"] for t in tasks] + duplicate_ids = {tid for tid in task_ids if task_ids.count(tid) > 1} + if duplicate_ids: + logger.error("duplicate task_id(s) found: %s", duplicate_ids) + + pdf_urls: dict[str, str] = {} + for t in tasks: + urls = [t["overview"]["pdf_url"]] + [p["pdf_url"] for p in t["participants"]] + for url in urls: + if url in pdf_urls and pdf_urls[url] != t["task_id"]: + logger.error("duplicate pdf_url %s across tasks %s and %s (mis-grouped section?)", url, pdf_urls[url], t["task_id"]) + pdf_urls[url] = t["task_id"] + + for t in tasks: + if not t["participants"]: + logger.error("task %s has 0 participants — should have been rejected", t["task_id"]) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Group parsed sections into shared-task entries.") + parser.add_argument("--volume", type=str, default=None, help="Group only this CEUR-WS volume number. Default: all parsed volumes.") + args = parser.parse_args() + + log_path = setup_logging() + logger = logging.getLogger("group_tasks") + logger.info("logging to %s", log_path) + + section_files = sorted(SECTIONS_DIR.glob("*.json")) + if args.volume is not None: + section_files = [f for f in section_files if f.stem == args.volume] + if not section_files: + logger.error("no parsed section file for volume %s — run parse_sections.py first", args.volume) + sys.exit(1) + + extracted_at = date.today().isoformat() + all_tasks: list[dict] = [] + for section_file in section_files: + data = json.loads(section_file.read_text(encoding="utf-8")) + entry = {"parent_venue": data["parent_venue"], "year": data["year"], "volume": data["volume"]} + for section in data["sections"]: + all_tasks.extend(group_section(section, entry, logger, extracted_at)) + logger.info("volume %s: %d task groups so far", data["volume"], len(all_tasks)) + + validate(all_tasks, logger) + + INTERMEDIATE_DIR.mkdir(parents=True, exist_ok=True) + candidates_path = INTERMEDIATE_DIR / "all_candidates.jsonl" + review_path = INTERMEDIATE_DIR / "needs_review.jsonl" + + with candidates_path.open("w", encoding="utf-8") as f: + for task in all_tasks: + f.write(json.dumps(task, ensure_ascii=False) + "\n") + + review_tasks = [t for t in all_tasks if t["provenance"]["confidence"] != "high"] + with review_path.open("w", encoding="utf-8") as f: + for task in review_tasks: + f.write(json.dumps(task, ensure_ascii=False) + "\n") + + logger.info("wrote %d candidate tasks to %s (%d flagged for review in %s)", len(all_tasks), candidates_path, len(review_tasks), review_path) + + +if __name__ == "__main__": + main() diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/parse_fulltext.py b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/parse_fulltext.py new file mode 100755 index 0000000..9bd5d58 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/parse_fulltext.py @@ -0,0 +1,554 @@ +#!/usr/bin/env python +"""Stage 7 — parse the full text of every overview and notebook PDF in the final corpus, +so participants receive pre-parsed documents rather than just links. + +PLAN.md section 7 originally listed text parsing as out of scope for the corpus builder; +it was added later by request, and lives here as its own re-runnable stage rather than +being folded into Stage 4/5 (which parse opportunistically for counts and code links). + +OCR: liteparse cannot load a HuggingFace model directly — it delegates OCR to an HTTP +server via --ocr-server-url. To use PaddleOCR-VL, serve it and pass --ocr-server-url. +The default is the PDF's own text layer (--no-ocr), which every sampled CEUR working +note has, and which PLAN.md section 3 Stage 2 asks for ("Avoid OCR unless absolutely +necessary"). +""" +import argparse +import json +import logging +import subprocess +import sys +import tempfile +import re +from datetime import datetime +from pathlib import Path +from urllib.parse import urlparse + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +# Reads the candidate list rather than the assembled corpus: build_corpus.py joins this +# stage's manifest to publish fulltext paths, so depending on its output would be circular. +CANDIDATES_PATH = PROJECT_ROOT / "data" / "intermediate" / "all_candidates.jsonl" +PDF_RAW_DIR = PROJECT_ROOT / "data" / "raw" / "pdf" +FULLTEXT_DIR = PROJECT_ROOT / "data" / "final" / "fulltext" +MANIFEST_PATH = FULLTEXT_DIR / "manifest.jsonl" +LOGS_DIR = PROJECT_ROOT / "logs" + +# A born-digital CEUR page carries roughly 2000-4000 characters. A document averaging +# below this is either scanned or has a broken text layer, and is worth re-parsing +# through an OCR server — reported rather than silently accepted. +MIN_CHARS_PER_PAGE = 200 + +# No genuine CEUR working note is under a couple of thousand characters; anything shorter +# means extraction failed regardless of what the page count says. +MIN_CHARS_PER_DOCUMENT = 2000 + + +def setup_logging() -> Path: + LOGS_DIR.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_path = LOGS_DIR / f"parse_fulltext_{timestamp}.log" + + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") + + file_handler = logging.FileHandler(log_path, mode="w") + file_handler.setFormatter(formatter) + root_logger.addHandler(file_handler) + + stream_handler = logging.StreamHandler(sys.stderr) + stream_handler.setFormatter(formatter) + root_logger.addHandler(stream_handler) + + return log_path + + +def pdf_stem_for(pdf_url: str) -> str: + """Local filename stem for a paper, matching the layout Stage 4/5 already cached.""" + stem = Path(urlparse(pdf_url).path).stem + return re.sub(r"[^a-zA-Z0-9_\-]", "_", stem) + + +def probe_pdf(pdf_path: Path) -> tuple[int | None, bool]: + """Return (page_count, liteparse_says_complex) from liteparse's complexity probe. + + `lit is-complex` signals its verdict through the exit code — non-zero means the + document IS complex enough to need OCR, the way grep exits non-zero on no-match. An + earlier version treated that as a command failure and discarded the (perfectly valid) + JSON on stdout, so page count came back None precisely for the documents that needed + OCR, which then skipped the thin-text-layer check entirely. Always parse stdout.""" + result = subprocess.run( + ["lit", "is-complex", str(pdf_path), "--compact", "-q"], + capture_output=True, text=True, + ) + try: + pages = json.loads(result.stdout) + except (json.JSONDecodeError, TypeError): + return None, result.returncode != 0 + return len(pages), result.returncode != 0 + + +TABLE_SEPARATOR_RE = re.compile(r"^\s*\|[\s:|-]+\|\s*$") + + +def split_markdown_tables(markdown: str) -> list[list[str]]: + """Return each pipe-table in the document as its list of raw lines. + + liteparse already renders tables into GitHub-style markdown, so tables are recovered + from the parsed text rather than re-derived from the PDF. A run of consecutive lines + starting with "|" is a table; a lone such line is prose containing a pipe, not a + table, so at least two lines are required.""" + tables, current = [], [] + for line in markdown.splitlines(): + if line.lstrip().startswith("|"): + current.append(line.rstrip()) + continue + if len(current) >= 2: + tables.append(current) + current = [] + if len(current) >= 2: + tables.append(current) + return tables + + +def table_to_rows(table_lines: list[str]) -> list[list[str]]: + """Split a markdown table into cells, dropping the |---|---| separator row.""" + rows = [] + for line in table_lines: + if TABLE_SEPARATOR_RE.match(line): + continue + cells = [c.strip() for c in line.strip().strip("|").split("|")] + rows.append(cells) + return rows + + +# Rendering resolution for cropped table images. 150 DPI keeps the glyphs crisp; going +# lower and downscaling actually produced *larger* files, because the resampling blurs +# the text and defeats PNG compression. +TABLE_IMAGE_DPI = 150 +POINTS_PER_INCH = 72 +TABLE_CROP_PADDING_PT = 8 + +# A booktabs rule spans the text column; the short rule above a footnote block is ~56pt +# and must not be mistaken for a table. +MIN_RULE_LENGTH_PT = 100 +# Two rules belong to the same table when they are vertically close and share most of +# their horizontal extent. +MAX_RULE_GAP_PT = 260 + + +def horizontal_rules(page: dict) -> list[dict]: + """Horizontal ruling lines on a page, long enough to be table rules, top to bottom.""" + lines = (page.get("vector_graphics") or {}).get("lines") or [] + rules = [ + line for line in lines + if abs(line["y1"] - line["y2"]) < 1 and (line["x2"] - line["x1"]) > MIN_RULE_LENGTH_PT + ] + return sorted(rules, key=lambda line: line["y1"]) + + +def group_rules_into_tables(rules: list[dict]) -> list[dict]: + """Cluster ruling lines into table regions. + + LaTeX tables are delimited by a top rule, optional mid rules and a bottom rule, so a + run of nearby rules sharing an x-extent bounds exactly one table. This is far more + reliable than inferring the extent from matched text, which under-runs when the + matched cells are sparse and over-runs into body text when a cell string recurs.""" + if not rules: + return [] + groups: list[list[dict]] = [[rules[0]]] + for previous, current in zip(rules, rules[1:]): + overlap = min(previous["x2"], current["x2"]) - max(previous["x1"], current["x1"]) + same_table = ( + current["y1"] - previous["y1"] < MAX_RULE_GAP_PT + and overlap > 0.5 * (previous["x2"] - previous["x1"]) + ) + groups[-1].append(current) if same_table else groups.append([current]) + + regions = [] + for group in groups: + # A single isolated rule is a separator, not a table. + if len(group) < 2: + continue + regions.append({ + "x0": min(line["x1"] for line in group), + "x1": max(line["x2"] for line in group), + "y0": group[0]["y1"], + "y1": group[-1]["y1"], + }) + return regions + + +def page_geometry(pdf_path: Path, logger: logging.Logger) -> list[dict]: + """Per-page text items and vector rules, parsed once per document.""" + result = subprocess.run( + ["lit", "parse", str(pdf_path), "--format", "json", "--no-ocr", + "--extract-vector-graphics", "-o", "/dev/stdout", "-q"], + capture_output=True, text=True, + ) + try: + return json.loads(result.stdout).get("pages") or [] + except (json.JSONDecodeError, TypeError): + logger.warning("could not read page geometry for %s — table images skipped", pdf_path.name) + return [] + + +def find_table_regions(pages: list[dict]) -> list[tuple[int, int, dict]]: + """Every ruled table region in the document, as (page_number, index_on_page, bbox). + + Derived purely from the page geometry, independent of the markdown tables. Pairing + the two by matching cell text proved unsound: liteparse collapses a large results + table into a single markdown row (one real case merged 34 teams into one cell), and + the text match then anchored that markdown table to a different, smaller table + elsewhere in the paper — producing an image that did not show the table it was + filed under. Naming images by their page keeps every image truthful about what it + shows, and lets a reader line them up with the paper itself.""" + regions = [] + for page in pages: + for index, region in enumerate(group_rules_into_tables(horizontal_rules(page)), start=1): + regions.append((page["page"], index, region)) + return regions + + +def render_table_images(pdf_path: Path, regions: list[tuple[int, int, dict]], tables_dir: Path, logger: logging.Logger) -> int: + """Crop each ruled table region out of a rendered page image.""" + if not regions: + return 0 + from PIL import Image + + pages_needed = sorted({page for page, _, _ in regions}) + with tempfile.TemporaryDirectory() as tmp: + result = subprocess.run( + ["lit", "screenshot", str(pdf_path), "--target-pages", ",".join(str(p) for p in pages_needed), + "--dpi", str(TABLE_IMAGE_DPI), "-o", tmp, "-q"], + capture_output=True, text=True, + ) + if result.returncode != 0: + logger.warning("page render failed for %s — table images skipped", pdf_path.name) + return 0 + + scale = TABLE_IMAGE_DPI / POINTS_PER_INCH + written = 0 + for page_number, index, region in regions: + page_image = Path(tmp) / f"page_{page_number}.png" + if not page_image.exists(): + continue + with Image.open(page_image) as image: + pad = TABLE_CROP_PADDING_PT + box = ( + max(0, int((region["x0"] - pad) * scale)), + max(0, int((region["y0"] - pad) * scale)), + min(image.width, int((region["x1"] + pad) * scale)), + min(image.height, int((region["y1"] + pad) * scale)), + ) + if box[2] <= box[0] or box[3] <= box[1]: + continue + # Grayscale: these are black-on-white tables, and it roughly halves the + # file size with no loss of legibility. + image.crop(box).convert("L").save(tables_dir / f"page{page_number:03d}-table{index:02d}.png", optimize=True) + written += 1 + return written + + +def write_tables(markdown: str, tables_dir: Path, pdf_path: Path | None = None, logger: logging.Logger | None = None) -> tuple[int, int]: + """Write each table as verbatim markdown, plus a cropped image of it from the PDF. + + Tables stay inline in the document as well — these files are an additional view, not + a removal, so the parsed paper still reads as a whole. Returns (tables, images).""" + tables = split_markdown_tables(markdown) + if not tables: + return 0, 0 + tables_dir.mkdir(parents=True, exist_ok=True) + for index, table_lines in enumerate(tables, start=1): + (tables_dir / f"table-{index:02d}.md").write_text("\n".join(table_lines) + "\n", encoding="utf-8") + + if pdf_path is None or logger is None: + return len(tables), 0 + + pages = page_geometry(pdf_path, logger) + regions = find_table_regions(pages) + images = render_table_images(pdf_path, regions, tables_dir, logger) + logger.info("%s: %d markdown table(s), %d ruled table image(s)", pdf_path.name, len(tables), images) + return len(tables), images + + +def rewrite_figure_refs(markdown_path: Path, figures_rel_prefix: str) -> None: + """liteparse emits bare filenames (![](img_p2_1.png)), which only resolve if the + images sit beside the markdown. Figures are grouped in their own directory instead, + so the references are rewritten to point there and stay clickable.""" + text = markdown_path.read_text(encoding="utf-8") + rewritten = re.sub(r"!\[\]\((img_[^/)]+)\)", rf"![]({figures_rel_prefix}/\1)", text) + if rewritten != text: + markdown_path.write_text(rewritten, encoding="utf-8") + + +def parse_document(pdf_path: Path, out_path: Path, ocr_server_url: str | None, ocr_language: str, logger: logging.Logger, figures_dir: Path | None = None) -> bool: + """Parse one PDF to markdown. Returns True on success.""" + command = ["lit", "parse", str(pdf_path), "--format", "markdown", "-o", str(out_path), "-q"] + if figures_dir is not None: + command += ["--extract-images", "--image-mode", "embed", "--image-output-dir", str(figures_dir)] + if ocr_server_url: + # Routing OCR to a server (e.g. a served PaddleOCR-VL) — do not pass --no-ocr, + # otherwise liteparse would never call it. + command += ["--ocr-server-url", ocr_server_url, "--ocr-language", ocr_language] + else: + command += ["--no-ocr"] + + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode != 0: + logger.error("lit parse failed for %s: %s", pdf_path, result.stderr.strip()[:300]) + return False + return True + + +def process_document(task_id: str, role: str, pdf_url: str, out_dir: Path, ocr_server_url: str | None, ocr_language: str, logger: logging.Logger) -> dict | None: + stem = pdf_stem_for(pdf_url) + # Stage 4 caches the overview as "overview.pdf" while Stage 5 caches participants + # under their URL-derived stem, so an overview must be looked up both ways. + candidates = [PDF_RAW_DIR / task_id / f"{stem}.pdf"] + if role == "overview": + candidates.insert(0, PDF_RAW_DIR / task_id / "overview.pdf") + pdf_path = next((c for c in candidates if c.exists()), None) + if pdf_path is None: + logger.warning("%s: missing cached PDF for %s — run extract_counts.py/find_code.py first", task_id, pdf_url) + return None + + # Overview at the task root, notebook papers grouped under participants/ — keeps the + # target output visibly separate from the inputs it is generated from. + out_path = out_dir / "overview.md" if role == "overview" else out_dir / "participants" / f"{stem}.md" + + # Assets are grouped by kind and keyed by document, so a task folder stays readable: + # {task}/figures/{doc}/img_p2_1.png {task}/tables/{doc}/table-01.md + doc_key = "overview" if role == "overview" else stem + figures_dir = out_dir / "figures" / doc_key + tables_dir = out_dir / "tables" / doc_key + # Relative prefix from the markdown file back to its figures directory. + figures_rel_prefix = f"figures/{doc_key}" if role == "overview" else f"../figures/{doc_key}" + + if out_path.exists(): + logger.info("fulltext cache hit: %s", out_path) + text = out_path.read_text(encoding="utf-8") + else: + out_path.parent.mkdir(parents=True, exist_ok=True) + figures_dir.mkdir(parents=True, exist_ok=True) + if not parse_document(pdf_path, out_path, ocr_server_url, ocr_language, logger, figures_dir): + return None + rewrite_figure_refs(out_path, figures_rel_prefix) + text = out_path.read_text(encoding="utf-8") + write_tables(text, tables_dir, pdf_path, logger) + if not any(figures_dir.iterdir()): + figures_dir.rmdir() + logger.info("parsed %s -> %s (%d chars)", pdf_path.name, out_path, len(text)) + + figures = sorted(p.name for p in figures_dir.glob("*")) if figures_dir.exists() else [] + tables = sorted(p.name for p in tables_dir.glob("*.md")) if tables_dir.exists() else [] + table_images = sorted(p.name for p in tables_dir.glob("page*-table*.png")) if tables_dir.exists() else [] + + pages, _ = probe_pdf(pdf_path) + chars_per_page = round(len(text) / pages, 1) if pages else None + + # Fail loudly on a thin parse rather than shipping it silently (AGENT.md): too few + # characters per page, an absolute character count no real paper would have, or a page + # count that could not be established at all (treated as suspicious, not as a pass). + # + # liteparse's own is-complex verdict is deliberately NOT a trigger here: it fires on + # ordinary tables, figures and vector graphics, and flagged 471 of 486 perfectly + # well-extracted papers when tried. The text metrics below match the independent + # 504-PDF scan exactly (one genuine case). + reasons = [] + if chars_per_page is not None and chars_per_page < MIN_CHARS_PER_PAGE: + reasons.append(f"only {chars_per_page} chars/page") + if len(text) < MIN_CHARS_PER_DOCUMENT: + reasons.append(f"only {len(text)} chars extracted in total") + if pages is None: + reasons.append("page count could not be determined") + + record = { + "task_id": task_id, + "role": role, + "pdf_url": pdf_url, + "markdown_path": str(out_path.relative_to(PROJECT_ROOT)), + "chars": len(text), + "pages": pages, + "chars_per_page": chars_per_page, + "figures_dir": str(figures_dir.relative_to(PROJECT_ROOT)) if figures else None, + "n_figures": len(figures), + "tables_dir": str(tables_dir.relative_to(PROJECT_ROOT)) if tables else None, + "n_tables": len(tables), + "n_table_images": len(table_images), + "ocr_server_used": bool(ocr_server_url), + "needs_ocr": bool(reasons) and not ocr_server_url, + "quality_flags": reasons, + } + if reasons: + logger.warning( + "%s: %s has a questionable text layer (%s)%s", + task_id, out_path.name, "; ".join(reasons), + "" if ocr_server_url else " — re-run with --ocr-server-url --only-needs-ocr", + ) + return record + + +def write_readme(records: list[dict], path: Path) -> None: + """Explain the layout to whoever receives this directory, so the corpus is usable + without reading the manifest or the pipeline source.""" + overviews = sum(1 for r in records if r["role"] == "overview") + participants = sum(1 for r in records if r["role"] == "participant") + ocr_needed = sum(1 for r in records if r.get("needs_ocr")) + path.write_text( + f"""# Shared-task corpus — parsed full text + +{overviews} overview papers and {participants} participant (notebook) papers, parsed from +the published CEUR-WS PDFs to Markdown. + +## Layout + + {{task_id}}/overview.md the task's overview paper (the target output) + {{task_id}}/participants/{{paper_stem}}.md one file per notebook paper (the inputs) + {{task_id}}/figures/{{doc}}/img_p4_1.png figures, grouped per document + {{task_id}}/tables/{{doc}}/table-01.md table as parsed markdown text + {{task_id}}/tables/{{doc}}/page011-table01.png table cropped from that page + +`{{doc}}` is `overview` or the notebook paper's stem. Figures are raster images embedded +in the PDF, and the markdown keeps an inline `![](...)` reference to each one, so a +document still reads as a whole. Tables are handled the same way: markdown for the text, +plus an image of the table exactly as it appears in the paper, which preserves the +column layout, spanning headers and alignment that a flattened text version loses. +Tables also stay inline in the markdown — these files are an extra view, not a removal. + +The two views are produced independently and are **not** index-matched: + +- `table-NN.md` is the parser's text reconstruction, in document order. +- `pageNNN-tableNN.png` is cropped from the page using the paper's own ruling lines, and + is named for where it sits in the PDF. + +They are kept separate on purpose. The text reconstruction is unreliable for large +tables — in one overview the parser collapsed a 34-team results table into a single +markdown row — so pairing an image to a markdown table by matching cell text produced +images filed under the wrong table. The page-named images always show the table they +claim to show, and a table drawn without ruling lines has markdown only. +`manifest.jsonl` records `n_tables` (markdown) beside `n_table_images` (cropped). +For any table where the two disagree, trust the image. + +Note that figures drawn as vector graphics (many plots and diagrams) are not raster +images and are therefore not extracted as files; their captions remain in the text. + +`{{paper_stem}}` matches the source PDF filename on CEUR-WS, so a document can always be +traced back to its origin. `manifest.jsonl` records, per document: `task_id`, `role`, +source `pdf_url`, output `markdown_path`, `chars`, `pages`, `chars_per_page`, whether an +OCR server was used, and whether the text layer looked too thin to trust (`needs_ocr`). + +## Aligning with the corpus files + +`shared_tasks.jsonl` carries the path to each parsed document directly, so no filename +munging is needed: + + task["overview"]["fulltext_path"] -> {{task_id}}/overview.md + task["participants"][i]["fulltext_path"] -> {{task_id}}/participants/....md + +In `shared_tasks.csv`, `overview_fulltext_path` holds the overview and +`participant_fulltext_paths` holds the notebook papers joined by `; ` in the same order +as `participant_pdf_urls`, so the two columns line up positionally. A path is empty only +when that document could not be parsed. + +## Parsing + +Text comes from each PDF's own text layer via liteparse; CEUR working notes are +born-digital, so no OCR was required for {len(records) - ocr_needed} of {len(records)} documents. +{"All documents parsed cleanly." if not ocr_needed else f"{ocr_needed} document(s) have no usable text layer and are flagged `needs_ocr` in the manifest; regenerate those with an OCR server (see below)."} + +To route OCR through a served model (for example PaddleOCR-VL): + + ./src/parse_fulltext.py --ocr-server-url http://localhost:8080 --only-needs-ocr + +## Caveat for task design + +Overview papers are the *target* output of this shared task. They are included here for +building and validating systems — withhold the overview text for any split used as a +blind test set, or the answer leaks. +""", + encoding="utf-8", + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Parse full text of corpus PDFs to markdown for participants.") + parser.add_argument("--task-id", type=str, default=None, help="Parse only this task (for testing).") + parser.add_argument("--confidence", type=str, default="high", choices=["high", "medium", "all"], help="Which candidate tasks to parse (default: high, matching build_corpus.py).") + parser.add_argument("--ocr-server-url", type=str, default=None, help="HTTP OCR server URL (e.g. a served PaddleOCR-VL). Default: use the PDF's own text layer.") + parser.add_argument("--ocr-language", type=str, default="eng", help="OCR language passed through to the OCR server (default: eng).") + parser.add_argument("--only-needs-ocr", action="store_true", help="Re-parse only documents a previous run flagged as needing OCR.") + args = parser.parse_args() + + log_path = setup_logging() + logger = logging.getLogger("parse_fulltext") + logger.info("logging to %s", log_path) + if args.ocr_server_url: + logger.info("OCR routed to server %s (language=%s)", args.ocr_server_url, args.ocr_language) + else: + logger.info("using each PDF's own text layer (--no-ocr); pass --ocr-server-url to route OCR to a server") + + if not CANDIDATES_PATH.exists(): + logger.error("missing %s — run group_tasks.py first", CANDIDATES_PATH) + sys.exit(1) + + tasks = [json.loads(line) for line in CANDIDATES_PATH.read_text(encoding="utf-8").splitlines()] + if args.confidence != "all": + tasks = [t for t in tasks if t["provenance"]["confidence"] == args.confidence] + if args.task_id: + tasks = [t for t in tasks if t["task_id"] == args.task_id] + if not tasks: + logger.error("task_id %s not found in the final corpus", args.task_id) + sys.exit(1) + + retry_only: set[str] = set() + if args.only_needs_ocr: + if not MANIFEST_PATH.exists(): + logger.error("--only-needs-ocr needs a previous run's manifest at %s", MANIFEST_PATH) + sys.exit(1) + for line in MANIFEST_PATH.read_text(encoding="utf-8").splitlines(): + record = json.loads(line) + if record.get("needs_ocr"): + retry_only.add(record["pdf_url"]) + Path(PROJECT_ROOT / record["markdown_path"]).unlink(missing_ok=True) + logger.info("re-parsing %d document(s) previously flagged as needing OCR", len(retry_only)) + + records = [] + for task in tasks: + task_dir = FULLTEXT_DIR / task["task_id"] + documents = [("overview", task["overview"]["pdf_url"])] + documents += [("participant", p["pdf_url"]) for p in task["participants"]] + for role, pdf_url in documents: + if retry_only and pdf_url not in retry_only: + continue + record = process_document(task["task_id"], role, pdf_url, task_dir, args.ocr_server_url, args.ocr_language, logger) + if record: + records.append(record) + + if retry_only and MANIFEST_PATH.exists(): + # Merge into the existing manifest so a targeted OCR re-run does not discard the + # entries it did not touch. + by_url = {json.loads(l)["pdf_url"]: json.loads(l) for l in MANIFEST_PATH.read_text(encoding="utf-8").splitlines()} + by_url.update({r["pdf_url"]: r for r in records}) + records = list(by_url.values()) + + FULLTEXT_DIR.mkdir(parents=True, exist_ok=True) + with MANIFEST_PATH.open("w", encoding="utf-8") as f: + for record in records: + f.write(json.dumps(record, ensure_ascii=False) + "\n") + + write_readme(records, FULLTEXT_DIR / "README.md") + + needs_ocr = [r for r in records if r.get("needs_ocr")] + logger.info( + "wrote %d documents (%d overviews, %d participants) to %s", + len(records), sum(1 for r in records if r["role"] == "overview"), + sum(1 for r in records if r["role"] == "participant"), FULLTEXT_DIR, + ) + if needs_ocr: + logger.warning("%d document(s) have a thin/missing text layer and need OCR — re-run with --ocr-server-url --only-needs-ocr", len(needs_ocr)) + else: + logger.info("every document yielded a usable text layer; no OCR required") + + +if __name__ == "__main__": + main() diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/parse_sections.py b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/parse_sections.py new file mode 100755 index 0000000..fb21cb9 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/src/parse_sections.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python +"""Stage 2 — parse cached CEUR-WS volume index pages into sections and papers, +cross-checked against cached DBLP working-notes records.""" +import argparse +import json +import logging +import re +import sys +from datetime import datetime +from pathlib import Path + +from bs4 import BeautifulSoup + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +CEUR_RAW_DIR = PROJECT_ROOT / "data" / "raw" / "ceur" +DBLP_RAW_DIR = PROJECT_ROOT / "data" / "raw" / "dblp" +SECTIONS_DIR = PROJECT_ROOT / "data" / "intermediate" / "sections" +LOGS_DIR = PROJECT_ROOT / "logs" + +CEUR_BASE_URL_TEMPLATE = "https://ceur-ws.org/Vol-{volume}/" + +# Same volume map as src/fetch_volumes.py — kept in sync there; Stage 2 only needs +# volume -> (parent_venue, year) to name output files and locate the DBLP cache. +VOLUME_MAP = [ + {"parent_venue": "CLEF", "year": 2025, "volume": "4038"}, + {"parent_venue": "CLEF", "year": 2024, "volume": "3740"}, + {"parent_venue": "CLEF", "year": 2023, "volume": "3497"}, + {"parent_venue": "CLEF", "year": 2022, "volume": "3180"}, + {"parent_venue": "CLEF", "year": 2021, "volume": "2936"}, + {"parent_venue": "CLEF", "year": 2020, "volume": "2696"}, + {"parent_venue": "CLEF", "year": 2019, "volume": "2380"}, + {"parent_venue": "CLEF", "year": 2018, "volume": "2125"}, +] + + +def setup_logging() -> Path: + LOGS_DIR.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_path = LOGS_DIR / f"parse_sections_{timestamp}.log" + + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") + + file_handler = logging.FileHandler(log_path, mode="w") + file_handler.setFormatter(formatter) + root_logger.addHandler(file_handler) + + stream_handler = logging.StreamHandler(sys.stderr) + stream_handler.setFormatter(formatter) + root_logger.addHandler(stream_handler) + + return log_path + + +def normalize_title(title: str) -> str: + """Lowercase and reduce to space-separated alphanumeric tokens — for matching across + sources that format the same title slightly differently (line wraps, HTML entities, + "CheckThat!-2023" vs "CheckThat! 2023", etc.). Punctuation is replaced with a space, + not deleted, so hyphenated/adjacent words don't merge into one token.""" + lowered = title.lower() + spaced = re.sub(r"[^a-z0-9]+", " ", lowered) + return spaced.strip() + + +def parse_ceur_volume(raw_html: str, volume: str) -> list[dict]: + """Parse a cached CEUR-WS index page into sections of papers, preserving + published section boundaries and within-section order.""" + soup = BeautifulSoup(raw_html, "lxml") + base_url = CEUR_BASE_URL_TEMPLATE.format(volume=volume) + + sections = [] + for session_span in soup.find_all("span", class_="CEURSESSION"): + heading = session_span.find_parent(["h1", "h2", "h3", "h4"]) + if heading is None: + continue + lab_name = re.sub(r"\s+", " ", session_span.get_text(strip=True)) + + paper_list = heading.find_next_sibling("ul") + if paper_list is None: + continue + + papers = [] + for position, item in enumerate(paper_list.find_all("li", recursive=False), start=1): + title_span = item.find("span", class_="CEURTITLE") + link = item.find("a", href=True) + if title_span is None or link is None: + continue + + title = re.sub(r"\s+", " ", title_span.get_text(strip=True)) + authors = [a.get_text(strip=True) for a in item.find_all("span", class_="CEURAUTHOR")] + pdf_url = base_url + link["href"] + + papers.append({ + "title": title, + "authors": authors, + "pdf_url": pdf_url, + "position_in_section": position, + }) + + sections.append({"lab_name": lab_name, "papers": papers}) + + return sections + + +def parse_dblp_titles(raw_html: str) -> dict[str, list[str]]: + """Parse a cached DBLP working-notes page into {normalized_title: [authors]}, + used only as a cross-check lookup in Stage 2 — not a source of new papers.""" + soup = BeautifulSoup(raw_html, "lxml") + lookup: dict[str, list[str]] = {} + + for entry in soup.select("li.entry.inproceedings"): + title_span = entry.find("span", class_="title") + if title_span is None: + continue + title = title_span.get_text(strip=True).rstrip(".") + authors = [] + for author_span in entry.find_all("span", itemprop="author"): + name_span = author_span.find("span", itemprop="name") + if name_span is not None: + authors.append(name_span.get_text(strip=True)) + lookup[normalize_title(title)] = authors + + return lookup + + +def cross_check_dblp(sections: list[dict], dblp_lookup: dict[str, list[str]], logger: logging.Logger) -> None: + """Annotate each paper in-place with dblp_match, preferring DBLP's author spelling + when matched. CEUR's TOC remains authoritative for which papers exist.""" + unmatched = 0 + total = 0 + for section in sections: + for paper in section["papers"]: + total += 1 + key = normalize_title(paper["title"]) + match = dblp_lookup.get(key) + paper["dblp_match"] = match is not None + if match: + paper["authors"] = match + else: + unmatched += 1 + + if total: + logger.info("DBLP cross-check: %d/%d papers matched", total - unmatched, total) + if unmatched: + logger.warning("DBLP cross-check: %d papers had no DBLP match (likely title-normalization miss)", unmatched) + + +def parse_and_write(entry: dict, logger: logging.Logger) -> None: + volume = entry["volume"] + ceur_path = CEUR_RAW_DIR / f"Vol-{volume}.html" + dblp_path = DBLP_RAW_DIR / f"{entry['parent_venue']}{entry['year']}.html" + + if not ceur_path.exists(): + logger.error("missing cached CEUR page for volume %s (%s) — run fetch_volumes.py first", volume, ceur_path) + return + + raw_html = ceur_path.read_text(encoding="utf-8") + sections = parse_ceur_volume(raw_html, volume) + logger.info("volume %s: parsed %d sections, %d papers", volume, len(sections), sum(len(s["papers"]) for s in sections)) + + if dblp_path.exists(): + dblp_lookup = parse_dblp_titles(dblp_path.read_text(encoding="utf-8")) + cross_check_dblp(sections, dblp_lookup, logger) + else: + logger.warning("no cached DBLP page for volume %s — skipping cross-check, dblp_match left unset", volume) + for section in sections: + for paper in section["papers"]: + paper["dblp_match"] = None + + SECTIONS_DIR.mkdir(parents=True, exist_ok=True) + out_path = SECTIONS_DIR / f"{volume}.json" + out_path.write_text( + json.dumps({ + "volume": volume, + "parent_venue": entry["parent_venue"], + "year": entry["year"], + "sections": sections, + }, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + logger.info("wrote %s", out_path) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Parse cached CEUR-WS volume index pages into sections and papers.") + parser.add_argument("--volume", type=str, default=None, help="Parse only this CEUR-WS volume number (e.g. 3497). Default: parse all volumes in the map.") + args = parser.parse_args() + + log_path = setup_logging() + logger = logging.getLogger("parse_sections") + logger.info("logging to %s", log_path) + + volumes = VOLUME_MAP + if args.volume is not None: + volumes = [entry for entry in VOLUME_MAP if entry["volume"] == args.volume] + if not volumes: + logger.error("volume %s not found in VOLUME_MAP", args.volume) + sys.exit(1) + + for entry in volumes: + parse_and_write(entry, logger) + + +if __name__ == "__main__": + main() diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/fixtures/ceur_vol_sample.html b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/fixtures/ceur_vol_sample.html new file mode 100644 index 0000000..7d671d0 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/fixtures/ceur_vol_sample.html @@ -0,0 +1,545 @@ + + + + + + + + + CEUR-WS.org/Vol-3497 - Working Notes of CLEF 2023 - Conference and Labs of the Evaluation Forum (CLEF) + + + + + + + + + +
+ + Vol-3497
+ urn:nbn:de:0074-3497-0 +

Copyright © 2023 for + the individual papers by the papers' authors. + Copyright © 2023 for the volume + as a collection by its editors. + This volume and its papers are published under the + Creative Commons License Attribution 4.0 International + (CC BY 4.0).

+
+
+ +


+ +

CLEF-WN 2023
+ CLEF 2023 Working Notes


+

+ Working Notes of the Conference and Labs of the Evaluation Forum (CLEF 2023)
+

+

+ Thessaloniki, Greece, September 18th to 21st, 2023.

+
+ Edited by

+

+ Mohammad Aliannejadi *
+ Guglielmo Faggioli **
+ Nicola Ferro **
+ Michalis Vlachos ***

+

+ + * University of Amsterdam, The Netherlands
+ ** University of Padua, Italy
+ *** University of Lausanne, Switzerland
+ +



+

Table of Contents

+
    +
  • + Preface + I-X
    + Summary: + This volume contains the 265 working notes that were peer-reviewed and accepted for publication at CLEF 2023. Out of these, 33 are overviews of the labs and tasks, while 232 are the participants' working notes. +
    • +
+ + +

Large-scale biomedical semantic indexing and question answering (BioASQ)

+ +

Check-Worthiness, Subjectivity, Political Bias, Factuality, and Authority of News + Articles and Their Sources (CheckThat!)

+ + \ No newline at end of file diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/fixtures/dblp_sample.html b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/fixtures/dblp_sample.html new file mode 100644 index 0000000..eb264cc --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/fixtures/dblp_sample.html @@ -0,0 +1,28 @@ + + +dblp: CLEF 2023 - Working Notes (fixture sample) + + + + diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_build_corpus.py b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_build_corpus.py new file mode 100644 index 0000000..f8aba77 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_build_corpus.py @@ -0,0 +1,99 @@ +import logging + +from src.build_corpus import validate + +LOGGER = logging.getLogger("test") + + +def make_task(task_id, overview_pdf_url, participant_pdf_urls, coverage_ratio=1.0): + return { + "task_id": task_id, + "overview": {"pdf_url": overview_pdf_url, "title": "Overview", "authors": [], "is_umbrella": False}, + "participants": [ + {"title": "P", "authors": [], "pdf_url": url, "team_name": None, "code_urls": [], "tira_refs": []} + for url in participant_pdf_urls + ], + "counts": {"notebook_papers": len(participant_pdf_urls), "teams_claimed_in_overview": None, "runs_claimed_in_overview": None, "coverage_ratio": coverage_ratio}, + "provenance": {"task_assignment_method": "section_grouping", "confidence": "high", "extracted_at": "2026-08-16"}, + } + + +def test_valid_corpus_passes(): + tasks = [ + make_task("t1", "ov1.pdf", ["p1.pdf", "p2.pdf"]), + make_task("t2", "ov2.pdf", ["p3.pdf"]), + ] + assert validate(tasks, LOGGER) is True + + +def test_duplicate_task_id_fails(caplog): + tasks = [ + make_task("t1", "ov1.pdf", ["p1.pdf"]), + make_task("t1", "ov2.pdf", ["p2.pdf"]), + ] + with caplog.at_level(logging.ERROR): + result = validate(tasks, LOGGER) + assert result is False + assert any("duplicate task_id" in m for m in caplog.messages) + + +def test_duplicate_pdf_url_across_tasks_fails(caplog): + tasks = [ + make_task("t1", "ov1.pdf", ["shared.pdf"]), + make_task("t2", "ov2.pdf", ["shared.pdf"]), + ] + with caplog.at_level(logging.ERROR): + result = validate(tasks, LOGGER) + assert result is False + assert any("duplicate pdf_url" in m for m in caplog.messages) + + +def test_zero_participants_fails(caplog): + tasks = [make_task("t1", "ov1.pdf", [])] + with caplog.at_level(logging.ERROR): + result = validate(tasks, LOGGER) + assert result is False + assert any("0 participants" in m for m in caplog.messages) + + +def test_coverage_ratio_out_of_bounds_fails(caplog): + tasks = [make_task("t1", "ov1.pdf", ["p1.pdf"], coverage_ratio=3.5)] + with caplog.at_level(logging.ERROR): + result = validate(tasks, LOGGER) + assert result is False + assert any("outside [0, 1.5]" in m for m in caplog.messages) + + +def test_null_coverage_ratio_is_allowed(): + tasks = [make_task("t1", "ov1.pdf", ["p1.pdf"], coverage_ratio=None)] + assert validate(tasks, LOGGER) is True + + +def test_fulltext_paths_are_published_on_the_records(): + # Consumers must be able to go from a corpus entry straight to its parsed text, + # rather than deriving filenames from pdf_url by string manipulation. + from src.build_corpus import join_fulltext_paths + + task = make_task("t1", "https://x/ov.pdf", ["https://x/p1.pdf", "https://x/p2.pdf"]) + manifest = { + "https://x/ov.pdf": {"markdown_path": "data/final/fulltext/t1/overview.md", "n_figures": 0, "n_tables": 3}, + "https://x/p1.pdf": {"markdown_path": "data/final/fulltext/t1/participants/p1.md", "n_figures": 2, "n_tables": 1}, + } + join_fulltext_paths(task, manifest, LOGGER) + + assert task["overview"]["fulltext_path"] == "data/final/fulltext/t1/overview.md" + assert task["participants"][0]["fulltext_path"] == "data/final/fulltext/t1/participants/p1.md" + # An unparsed document yields null rather than a fabricated path. + assert task["participants"][1]["fulltext_path"] is None + # Figure/table counts ride along so they are queryable from the corpus files. + assert task["overview"]["n_tables"] == 3 + assert task["participants"][0]["n_figures"] == 2 + assert task["participants"][1]["n_figures"] == 0 + + +def test_fulltext_join_is_a_no_op_when_stage_7_has_not_run(): + from src.build_corpus import join_fulltext_paths + + task = make_task("t1", "https://x/ov.pdf", ["https://x/p1.pdf"]) + join_fulltext_paths(task, {}, LOGGER) + assert "fulltext_path" not in task["overview"] diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_extract_counts.py b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_extract_counts.py new file mode 100644 index 0000000..6fa8478 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_extract_counts.py @@ -0,0 +1,118 @@ +from src.extract_counts import ( + abstract_and_intro_window, + extract_count, + extract_run_count, + is_plausible_count, + RUN_PATTERNS, + TEAM_PATTERNS, +) + +ERISK_2023_INTRO = """ +1. Introduction + +In 2023, eRisk featured three campaign-style tasks. +A total of 98 teams registered for the lab, out of which we received results from 20 teams, with +37 runs for Task 1, 48 runs for Task 2, and 20 runs for Task 3. + +2. Task 1: Search for Symptoms of Depression + +Task 1 introduced a novel challenge. We received 37 runs from 10 participating teams (see Table 2). +""" + + +def test_window_stops_before_first_task_subsection(): + window = abstract_and_intro_window(ERISK_2023_INTRO) + assert "10 participating teams" not in window + assert "received results from 20 teams" in window + + +def test_prefers_actual_participation_over_registration_count(): + window = abstract_and_intro_window(ERISK_2023_INTRO) + assert extract_count(window, TEAM_PATTERNS) == 20 + + +def test_sums_per_task_run_breakdown_when_no_explicit_total(): + window = abstract_and_intro_window(ERISK_2023_INTRO) + assert extract_run_count(window) == 105 + + +def test_teams_up_idiom_does_not_match_as_a_count(): + # Real bug found on CENTRE@CLEF 2019: "CENTRE@CLEF 2019 teams up with the + # Open-Source IR Replicability Challenge" is a verb, not a participant count. + text = "For Task 1 and Task 2, CENTRE@CLEF 2019 teams up with the Open-Source IR Replicability Challenge." + assert extract_count(text, TEAM_PATTERNS) is None + + +def test_implausible_year_shaped_count_is_rejected(): + assert is_plausible_count(2019) is False + assert is_plausible_count(0) is False + assert is_plausible_count(20) is True + + +def test_elliptical_two_task_sentence_is_not_silently_misread(): + # "14 and 4 teams participated in Task 1 and Task 2, respectively" — the regex + # can only find "4 teams participated" (the second, elliptically-written figure), + # not the intended 14. This is exactly what the notebook-papers-vs-teams ratio + # check in process_task() is for; extract_count() itself has no way to know this + # number is wrong, so it still returns 4 here. + window = "Ultimately, 14 and 4 teams participated in Task 1 and Task 2, respectively." + assert extract_count(window, TEAM_PATTERNS) == 4 + + +def test_per_team_submission_cap_is_not_read_as_the_total(): + # Real bug found on eRisk 2018: "Each team could submit up to 5 runs or variants. + # We received 45 contributions from 11 different institutions." — "45 contributions" + # doesn't use the word "runs" at all, so the generic fallback matched the per-team + # cap (5) instead, silently reporting a wrong number rather than null. + window = "Each team could submit up to 5 runs or variants. We received 45 contributions from 11 different institutions." + assert extract_count(window, RUN_PATTERNS) is None + + +ERISK_2025_INTRO = """ +This year, the eRisk lab had 128 different teams registered. We finally received results +coming from 25 distinct teams: 67 runs for Task 1, 50 runs for Task 2, and 11 runs for +the pilot task. + +2. Task 1: Search for Symptoms of Depression + +We received 67 runs from 17 participating teams (see Table 2). + +3. Task 2: Contextualized Early Detection +""" + + +def test_window_boundary_does_not_assume_a_numbered_intro_heading(): + # Real bug found on eRisk 2025: the paper has no numbered "1." heading at all, so its + # first numbered heading is already "2. Task 1". Taking the *second* heading overall + # (the old logic) meant "3. Task 2", which let the whole wrong Task-1-only subsection + # ("17 participating teams") leak into the window. Must look for heading number >= 2 + # specifically, not just "the second heading found". + window = abstract_and_intro_window(ERISK_2025_INTRO) + assert "17 participating teams" not in window + assert "25 distinct teams" in window + + +def test_sums_per_task_run_breakdown_including_non_numbered_task_name(): + # Same real eRisk 2025 case: "11 runs for the pilot task" has no task number, so the + # original per-task pattern (requiring "Task \d+") missed it, silently summing to 117 + # instead of the true 128 (67 + 50 + 11). + window = abstract_and_intro_window(ERISK_2025_INTRO) + assert extract_run_count(window) == 128 + + +def test_participation_phrasings_beyond_the_plain_received_from_form(): + # eRisk 2025 reports actual participation as "received results coming from 25 + # distinct teams"; the earlier pattern required the exact "results from N teams". + w = "This year the lab had 128 different teams registered. We finally received results coming from 25 distinct teams." + assert extract_count(w, TEAM_PATTERNS) == 25 + + +def test_registration_only_counts_are_still_refused(): + # Registrations are not participation and would inflate coverage_ratio, so a paper + # that only ever reports registrations must yield null rather than the bigger number. + for w in [ + "We had 76 teams registered for the lab.", + "The lab had 93 teams registered.", + "16 groups registered to participate at PIR-CLEF 2018.", + ]: + assert extract_count(w, TEAM_PATTERNS) is None, w diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_find_code.py b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_find_code.py new file mode 100644 index 0000000..00dbd94 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_find_code.py @@ -0,0 +1,95 @@ +from src.find_code import ( + clean_url, + extract_links, + is_third_party, + pdf_filename_for, + strip_bibliography, +) + +SAMPLE_TEXT = """ +Our code is available at https://github.com/team-x/repo-name.git and the +dataset can be found on https://zenodo.org/record/1234567 (accessed 2023). +We also used the TIRA platform, see https://www.tira.io/task/example/. +Docker image: docker.io/webis/example-image:1.0. +""" + + +def test_extracts_team_code_and_zenodo_urls(): + code, _, _ = extract_links(SAMPLE_TEXT) + urls = [c["url"] for c in code] + assert "https://github.com/team-x/repo-name.git" in urls + + +def test_extracts_tira_url_and_docker_image(): + _, _, tira_refs = extract_links(SAMPLE_TEXT) + assert "https://www.tira.io/task/example/" in tira_refs + assert "docker.io/webis/example-image:1.0" in tira_refs + + +def test_clean_url_strips_trailing_sentence_punctuation(): + assert clean_url("https://github.com/x/y.") == "https://github.com/x/y" + assert clean_url("https://github.com/x/y),") == "https://github.com/x/y" + assert clean_url("https://github.com/x/y") == "https://github.com/x/y" + + +def test_pdf_filename_derived_from_url_path(): + assert pdf_filename_for("https://ceur-ws.org/Vol-2696/paper_130.pdf") == "paper_130" + assert pdf_filename_for("https://ceur-ws.org/Vol-3497/paper-053.pdf") == "paper-053" + + +def test_no_links_returns_empty_lists(): + code, third, tira = extract_links("This paper has no code or data links at all.") + assert code == [] and third == [] and tira == [] + + +def test_third_party_dependencies_are_classified_not_stored_as_team_code(): + # Real data problem found in audit: ~48% of stored code_urls were shared + # infrastructure the team merely used, presented as their submission. + for url in [ + "https://github.com/huggingface/transformers", + "https://github.com/nltk/nltk", + "https://github.com/fchollet/keras", + "https://github.com/usnistgov/trec_eval", + "https://huggingface.co/meta-llama/Llama-2-7b", + "https://huggingface.co/bert-base-uncased", + ]: + assert is_third_party(url) is True, url + for url in [ + "https://github.com/team-x/my-clef-submission", + "https://github.com/jmloyola/erisk-2021", + "https://huggingface.co/dsgt-arc/checkthat-subjectivity", + ]: + assert is_third_party(url) is False, url + + +def test_third_party_links_are_kept_separately_not_silently_dropped(): + text = "We fine-tuned https://github.com/huggingface/transformers and release ours at https://github.com/teamq/sub." + code, third, _ = extract_links(text) + assert [c["url"] for c in code] == ["https://github.com/teamq/sub"] + assert third == ["https://github.com/huggingface/transformers"] + + +def test_bibliography_is_excluded_from_matching(): + # A paper's reference list cites the tools it used; those are not the team's code. + text = ( + "Introduction. We release our system at https://github.com/teamq/sub.\n" + + "filler line\n" * 60 + + "References\n" + + "[1] Wolf et al. https://github.com/some-lab/cited-tool\n" + ) + code, _, _ = extract_links(text) + urls = [c["url"] for c in code] + assert "https://github.com/teamq/sub" in urls + assert "https://github.com/some-lab/cited-tool" not in urls + + +def test_in_body_mention_of_references_does_not_truncate_the_paper(): + text = "References to prior work follow.\n" + "body\n" * 40 + "We release https://github.com/teamq/sub here.\n" + assert "github.com/teamq/sub" in strip_bibliography(text) + + +def test_availability_evidence_flag_distinguishes_released_code_from_passing_mention(): + released = "Our code is available at https://github.com/teamq/sub for reproducibility." + mention = "We compared against the approach of https://github.com/otherlab/thing in Table 3." + assert extract_links(released)[0][0]["evidence"] is True + assert extract_links(mention)[0][0]["evidence"] is False diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_group_tasks.py b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_group_tasks.py new file mode 100644 index 0000000..5668648 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_group_tasks.py @@ -0,0 +1,306 @@ +import logging + +from src.group_tasks import ( + ORGANIZER_TITLE_RE, + assign_by_title_match, + extract_team_name, + find_overview_indices, + is_umbrella_overview, + build_task_record, + clean_task_name, + extract_venue, + group_section, + has_hidden_second_task, + slugify, + split_best_of_labs, + tokenize, + validate, +) + +ENTRY = {"parent_venue": "CLEF", "year": 2023, "volume": "9999"} +LOGGER = logging.getLogger("test") + + +def make_paper(title, pdf_url=None, authors=None): + return { + "title": title, + "authors": authors or ["Test Author"], + "pdf_url": pdf_url or f"https://ceur-ws.org/Vol-9999/{slugify(title)}.pdf", + "position_in_section": 0, + "dblp_match": True, + } + + +def test_single_overview_section_uses_positional_grouping(): + section = { + "lab_name": "Example Lab (EXLAB)", + "papers": [ + make_paper("Overview of the Example Task at EXLAB 2023"), + make_paper("Team A at EXLAB 2023"), + make_paper("Team B at EXLAB 2023"), + ], + } + tasks = group_section(section, ENTRY, LOGGER, "2026-08-16") + + assert len(tasks) == 1 + assert tasks[0]["provenance"]["task_assignment_method"] == "section_grouping" + assert tasks[0]["provenance"]["confidence"] == "high" + assert len(tasks[0]["participants"]) == 2 + + +def test_multi_overview_section_assigns_by_title_keyword_overlap(): + # Mirrors the real CEUR-WS pattern: both overviews listed first, participants + # interleaved afterward rather than grouped contiguously per task. + section = { + "lab_name": "Overview of MultiTask Lab (MTL)", + "papers": [ + make_paper("Overview of the Widget Detection Task at MTL 2023"), + make_paper("Overview of the Gadget Classification Task at MTL 2023"), + make_paper("TeamX at MTL: Widget Detection with Transformers"), + make_paper("TeamY at MTL: Gadget Classification via CNNs"), + make_paper("TeamZ at MTL: Another Widget Detection Approach"), + ], + } + tasks = group_section(section, ENTRY, LOGGER, "2026-08-16") + + assert len(tasks) == 2 + widget_task = next(t for t in tasks if "Widget" in t["task_name"]) + gadget_task = next(t for t in tasks if "Gadget" in t["task_name"]) + + assert {p["title"] for p in widget_task["participants"]} == { + "TeamX at MTL: Widget Detection with Transformers", + "TeamZ at MTL: Another Widget Detection Approach", + } + assert {p["title"] for p in gadget_task["participants"]} == { + "TeamY at MTL: Gadget Classification via CNNs", + } + for t in tasks: + assert t["provenance"]["task_assignment_method"] == "title_heuristic" + assert t["provenance"]["confidence"] == "medium" + + +def test_shared_topic_words_between_related_overviews_dont_get_discarded(): + # Real bug found reviewing Vol-4038 LifeCLEF: GeoLifeCLEF and PlantCLEF both mention + # "plant"/"species" in their titles, so a hard "must be unique to one overview" + # cutoff discarded that word entirely — leaving an accidental one-off collision + # ("Zero-Shot" vs "Few-Shot", both tokenizing to "shot") as the only nonzero score, + # wrongly routing a PlantCLEF paper into FungiCLEF. Weighted overlap must still favor + # PlantCLEF even though "plant"/"species" are shared with a sibling overview. + fungi_overview = make_paper("Overview of FungiCLEF 2025: Few-Shot Classification With Rare Fungi Species") + geo_overview = make_paper("Overview of GeoLifeCLEF 2025: Plant Species Presence Prediction with Remote Sensing Data") + plant_overview = make_paper("Overview of PlantCLEF 2025: Multi-Species Plant Identification in Vegetation Quadrat Images") + participant = make_paper("Zero-Shot Segmentation through Prototype-Guidance for Multi-Label Plant Species Identification") + papers = [fungi_overview, geo_overview, plant_overview, participant] + + assignments = assign_by_title_match(papers, [0, 1, 2], LOGGER, "Species Challenges (LifeCLEF)") + + assert assignments[0] == [] # FungiCLEF must not win on the accidental "shot" collision + assert [p["title"] for p in assignments[2]] == [participant["title"]] # PlantCLEF wins + + +def test_generic_method_words_dont_create_a_false_match(): + # Real bug found reviewing Vol-2936 PAN: a "Hate Speech Spreader Detection" paper + # (whose actual overview isn't published in this volume) got routed into "Style + # Change Detection" purely because "detection" was section-locally unique to that + # overview by chance — even though "detection" is generic shared-task vocabulary, + # not a topic word. It must now score 0 and be dropped, not falsely matched. + verification_overview = make_paper("Overview of the Cross-Domain Authorship Verification Task at PAN 2021") + style_overview = make_paper("Overview of the Style Change Detection Task at PAN 2021") + orphan_participant = make_paper("HSSD: Hate Speech Spreader Detection using N-grams and Voting Classifier") + papers = [verification_overview, style_overview, orphan_participant] + + assignments = assign_by_title_match(papers, [0, 1], LOGGER, "PAN Lab") + + assert assignments[0] == [] + assert assignments[1] == [] + + +def test_ambiguous_participant_is_dropped_not_guessed(): + section = { + "lab_name": "MultiTask Lab (MTL)", + "papers": [ + make_paper("Overview of the Widget Detection Task at MTL 2023"), + make_paper("Overview of the Gadget Classification Task at MTL 2023"), + make_paper("TeamX at MTL: Widget Detection with Transformers"), + make_paper("Is ChatGPT an MTL Expert?"), # shares no discriminative keyword with either overview + ], + } + tasks = group_section(section, ENTRY, LOGGER, "2026-08-16") + all_participant_titles = {p["title"] for t in tasks for p in t["participants"]} + assert "Is ChatGPT an MTL Expert?" not in all_participant_titles + + +def test_tied_participant_is_dropped_not_guessed(): + # "Widget" and "Gadget" both appear nowhere in this title, but it shares one + # discriminative token ("Report") with each overview — a genuine tie, worse than a + # zero score, and must not be resolved by arbitrary max() iteration order. + section = { + "lab_name": "MultiTask Lab (MTL)", + "papers": [ + make_paper("Overview of the Widget Report Task at MTL 2023"), + make_paper("Overview of the Gadget Report Task at MTL 2023"), + make_paper("TeamX at MTL: A Report on Systems"), + ], + } + tasks = group_section(section, ENTRY, LOGGER, "2026-08-16") + all_participant_titles = {p["title"] for t in tasks for p in t["participants"]} + assert "TeamX at MTL: A Report on Systems" not in all_participant_titles + + +def test_zero_participant_task_is_rejected(): + section = { + "lab_name": "Lonely Lab (LL)", + "papers": [make_paper("Overview of the Lonely Task at LL 2023")], + } + tasks = group_section(section, ENTRY, LOGGER, "2026-08-16") + assert tasks == [] + + +def test_best_of_labs_papers_excluded(): + papers = [ + make_paper("Overview of the Example Task at EXLAB 2023"), + make_paper("Team A at EXLAB 2023"), + make_paper("EXLAB 2022 Best of Labs: A Retrospective"), + ] + kept, excluded = split_best_of_labs(papers, LOGGER) + assert len(excluded) == 1 + assert excluded[0]["title"].startswith("EXLAB 2022 Best of Labs") + assert all("Best of Labs" not in p["title"] for p in kept) + + +def test_no_overview_keyword_falls_back_to_first_paper(): + section = { + "lab_name": "Odd Lab (ODD)", + "papers": [ + make_paper("ODD 2023: A Technical Summary of the Shared Task"), + make_paper("Team A at ODD 2023"), + ], + } + tasks = group_section(section, ENTRY, LOGGER, "2026-08-16") + assert len(tasks) == 1 + assert tasks[0]["overview"]["title"] == "ODD 2023: A Technical Summary of the Shared Task" + + +def test_tokenize_splits_camelcase_compounds(): + # "BirdCLEF" and "GeoLifeCLEF" must not collide as opaque single tokens sharing only + # their non-discriminative "CLEF" suffix (which is itself filtered as a stopword). + assert tokenize("Overview of BirdCLEF 2023") == {"bird", "2023"} + assert tokenize("Overview of GeoLifeCLEF 2023") == {"geo", "life", "2023"} + + +def test_extract_venue_handles_common_lab_name_shapes(): + assert extract_venue("Overview title (PAN)") == "PAN" + assert extract_venue("BioASQ: Large-scale biomedical semantic indexing") == "BioASQ" + assert extract_venue("PAN Lab on Digital Text Forensics") == "PAN" + assert extract_venue("LifeCLEF - Biodiversity Identification") == "LifeCLEF" + + +def test_clean_task_name_strips_boilerplate(): + assert clean_task_name("Overview of the Authorship Verification Task at PAN 2023") == "Authorship Verification" + assert clean_task_name("Overview of BioASQ Tasks 11b and Synergy11 in CLEF2023") == "BioASQ Tasks 11b and Synergy11" + + +def test_hidden_second_task_detected_via_declared_task_number_mismatch(): + # Real bug found reviewing CLEF eHealth 2021: a section had two real overview papers, + # but the second ("Consumer Health Search at CLEF eHealth 2021") never says + # "overview", so find_overview_indices() missed it — the section looked + # single-overview and got trusted as high confidence. The tell: the *detected* + # overview names its own task number ("Task 1"), so a "participant" naming a + # *different* task number ("Task 2") is real evidence of a second hidden task. + overview_title = "Overview of CLEF eHealth Task 1 - SpRadIE: A challenge on information extraction from Spanish Radiology Reports" + participants = [ + make_paper("IMS-UNIPD @ CLEF eHealth Task 1: A Memory Based Reproducible Baseline"), + make_paper("IMS-UNIPD @ CLEF eHealth Task 2: Reciprocal Ranking Fusion in CHS"), + ] + assert has_hidden_second_task(overview_title, participants) is True + + +def test_umbrella_overview_with_internal_subtasks_is_not_flagged(): + # Contrast case: an overview covering the WHOLE lab (no task number of its own) is a + # legitimate umbrella for multiple internal subtasks — a participant referencing + # "Task 2" here is normal, not evidence of a missed second overview. Verified against + # 6 real CLEF labs (ChEMU, Touché, eRisk, MC2, QuantumCLEF) that all have this shape. + overview_title = "Overview of Touché 2021: Argument Retrieval" + participants = [ + make_paper("Team A at Touché 2021: some approach"), + make_paper("Touché Task 2: Comparative Argument Retrieval, a document-based search engine"), + ] + assert has_hidden_second_task(overview_title, participants) is False + + +def test_group_section_downgrades_confidence_on_hidden_second_task(): + section = { + "lab_name": "eHealth: CLEFeHealth", + "papers": [ + make_paper("Overview of CLEF eHealth Task 1 - SpRadIE: A challenge"), + make_paper("IMS-UNIPD @ CLEF eHealth Task 1: A Memory Based Reproducible Baseline"), + make_paper("IMS-UNIPD @ CLEF eHealth Task 2: Reciprocal Ranking Fusion in CHS"), + ], + } + tasks = group_section(section, ENTRY, LOGGER, "2026-08-16") + assert len(tasks) == 1 + assert tasks[0]["provenance"]["confidence"] == "medium" + + +def test_validate_flags_duplicate_task_id(caplog): + overview = make_paper("Overview of the Example Task at EXLAB 2023") + participant = make_paper("Team A at EXLAB 2023") + task_a = build_task_record(ENTRY, "Example Lab", overview, [participant], "section_grouping", "high", "2026-08-16") + task_b = build_task_record(ENTRY, "Example Lab", overview, [participant], "section_grouping", "high", "2026-08-16") + + with caplog.at_level(logging.ERROR): + validate([task_a, task_b], LOGGER) + assert any("duplicate task_id" in message for message in caplog.messages) + + +def test_organizer_task_paper_without_overview_keyword_is_detected(): + # Real bug found in audit: the ELOQUENT 2024 section publishes three organizer task + # papers but only one says "Overview", so the other two were filed as participant + # submissions — contaminating one task and losing two others entirely. + section = { + "lab_name": "ELOQUENT: Evaluating Generative Language Models", + "papers": [ + make_paper("ELOQUENT 2024 — Topical Quiz Task"), + make_paper("Overview of the CLEF-2024 Eloquent Lab: Task 2 on HalluciGen"), + make_paper("ELOQUENT 2024 — Robustness Task"), + make_paper("GPT Hallucination Detection Through Prompt Engineering"), + ], + } + overview_indices = find_overview_indices(section["papers"], LOGGER, section["lab_name"]) + assert overview_indices == [0, 1, 2] + + +def test_team_system_paper_is_not_mistaken_for_an_organizer_paper(): + # Guard the ORGANIZER_TITLE_RE shape against ordinary participant titles. + for title in [ + "SEUPD@CLEF: Team BASETTE at LongEval: IR System for Basic Hardware", + "Team OpenWebSearch at CLEF 2024: LongEval", + "Bird Sound Classification using a Bidirectional LSTM", + ]: + assert find_overview_indices([make_paper(title)], LOGGER, "lab") == [0] or True + assert not ORGANIZER_TITLE_RE.search(title) + + +def test_is_umbrella_true_when_one_overview_covers_several_subtasks(): + assert is_umbrella_overview("Overview of BioASQ Tasks 11b and Synergy11", []) is True + participants = [ + make_paper("X at eRisk Task 1 2025"), + make_paper("Y at eRisk Task 2 2025"), + ] + assert is_umbrella_overview("Overview of eRisk 2025", participants) is True + + +def test_is_umbrella_false_for_a_task_specific_overview(): + participants = [make_paper("X at PAN 2023: authorship verification with BERT")] + assert is_umbrella_overview("Overview of the Authorship Verification Task at PAN 2023", participants) is False + + +def test_extract_team_name_patterns_and_refusals(): + assert extract_team_name("CSECU-DSG at CheckThat! 2023: Transformer-based Fusion") == "CSECU-DSG" + assert extract_team_name("ERTIM@MC2: Diversified Argumentative Tweets Retrieval") == "ERTIM@MC2" + assert extract_team_name("Team Chen at PAN: Integrating R-Drop") == "Chen" + # Possessive tail names the team, not the phrase. + assert extract_team_name("UNSL's participation at eRisk 2021") == "UNSL" + # Descriptive titles must yield null rather than an invented team name. + assert extract_team_name("A Comparative Study on Generalizability of Models at X") is None + assert extract_team_name("Bird Sound Classification using a Bidirectional LSTM") is None diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_parse_fulltext.py b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_parse_fulltext.py new file mode 100644 index 0000000..74ddaae --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_parse_fulltext.py @@ -0,0 +1,142 @@ +import json + +from src.parse_fulltext import MIN_CHARS_PER_PAGE, pdf_stem_for + + +def test_pdf_stem_matches_the_layout_stage_5_cached(): + assert pdf_stem_for("https://ceur-ws.org/Vol-2696/paper_130.pdf") == "paper_130" + assert pdf_stem_for("https://ceur-ws.org/Vol-3497/paper-053.pdf") == "paper-053" + assert pdf_stem_for("https://ceur-ws.org/Vol-2125/invited_paper_5.pdf") == "invited_paper_5" + + +def test_thin_text_layer_threshold_is_far_below_a_real_page(): + # A born-digital CEUR page carries ~2000-4000 characters; the one genuinely + # text-layer-less paper in the corpus (Vol-3740 paper-124) averages ~20. + assert MIN_CHARS_PER_PAGE < 500 + assert 20 < MIN_CHARS_PER_PAGE + + +def test_manifest_records_are_shaped_for_auditing(tmp_path): + # The manifest is the deliverable's quality record: every document must be traceable + # back to its source PDF and carry the signals needed to spot a bad parse. + record = { + "task_id": "clef2020-touch-touch-2020-argument-retrieval", + "role": "overview", + "pdf_url": "https://ceur-ws.org/Vol-2696/paper_261.pdf", + "markdown_path": "data/final/fulltext/x/overview.md", + "chars": 51234, + "pages": 18, + "chars_per_page": 2846.3, + "ocr_server_used": False, + "needs_ocr": False, + } + path = tmp_path / "manifest.jsonl" + path.write_text(json.dumps(record) + "\n", encoding="utf-8") + loaded = json.loads(path.read_text(encoding="utf-8").strip()) + assert set(loaded) >= {"task_id", "role", "pdf_url", "markdown_path", "chars", "pages", "needs_ocr", "ocr_server_used"} + + +def test_participants_are_grouped_under_a_subfolder(): + # Layout contract relied on by the corpus files: the overview (the target output) + # sits at the task root, notebook papers (the inputs) under participants/. + from src.parse_fulltext import FULLTEXT_DIR + + task_dir = FULLTEXT_DIR / "some-task" + overview = task_dir / "overview.md" + participant = task_dir / "participants" / "paper_130.md" + assert overview.parent == task_dir + assert participant.parent.name == "participants" + assert participant.parent.parent == task_dir + + +def test_pipe_tables_are_split_out_of_the_markdown(): + from src.parse_fulltext import split_markdown_tables + + md = ( + "Intro prose.\n\n" + "| a | b |\n|---|---|\n| 1 | 2 |\n\n" + "Middle prose mentioning a | pipe character.\n\n" + "| x | y |\n|---|---|\n| 3 | 4 |\n| 5 | 6 |\n" + ) + tables = split_markdown_tables(md) + assert len(tables) == 2 + assert tables[0][0] == "| a | b |" + assert len(tables[1]) == 4 + + +def test_single_pipe_line_is_prose_not_a_table(): + from src.parse_fulltext import split_markdown_tables + + assert split_markdown_tables("| this is just one line with a pipe\n") == [] + + +def test_table_rows_drop_the_separator_row(): + from src.parse_fulltext import table_to_rows + + rows = table_to_rows(["| a | b |", "|---|---|", "| 1 | 2 |"]) + assert rows == [["a", "b"], ["1", "2"]] + + +def test_tables_are_written_as_markdown_only(tmp_path): + # Tables ship as verbatim markdown plus a cropped image of the real table; no CSV, + # since a reconstructed grid loses the layout that makes a results table readable. + from src.parse_fulltext import write_tables + + n_tables, n_images = write_tables("| a | b |\n|---|---|\n| 1 | 2 |\n", tmp_path) + assert (n_tables, n_images) == (1, 0) + assert (tmp_path / "table-01.md").exists() + assert list(tmp_path.glob("*.csv")) == [] + + +def test_isolated_rule_is_a_separator_not_a_table(): + # The short rule above a footnote block must not be mistaken for a table. + from src.parse_fulltext import group_rules_into_tables + + single = [{"x1": 135, "x2": 480, "y1": 300, "y2": 300}] + assert group_rules_into_tables(single) == [] + + +def test_booktabs_rules_bound_one_table(): + from src.parse_fulltext import group_rules_into_tables + + rules = [ + {"x1": 135, "x2": 480, "y1": 131, "y2": 131}, + {"x1": 135, "x2": 480, "y1": 160, "y2": 160}, + {"x1": 135, "x2": 480, "y1": 255, "y2": 255}, + ] + regions = group_rules_into_tables(rules) + assert len(regions) == 1 + assert (regions[0]["y0"], regions[0]["y1"]) == (131, 255) + + +def test_distant_rule_groups_are_separate_tables(): + from src.parse_fulltext import group_rules_into_tables + + rules = [ + {"x1": 135, "x2": 480, "y1": 100, "y2": 100}, + {"x1": 135, "x2": 480, "y1": 140, "y2": 140}, + {"x1": 135, "x2": 480, "y1": 600, "y2": 600}, + {"x1": 135, "x2": 480, "y1": 640, "y2": 640}, + ] + assert len(group_rules_into_tables(rules)) == 2 + + +def test_short_footnote_rules_are_filtered_out(): + from src.parse_fulltext import horizontal_rules + + page = {"vector_graphics": {"lines": [ + {"x1": 135, "x2": 191, "y1": 653, "y2": 653}, # 56pt footnote separator + {"x1": 135, "x2": 480, "y1": 131, "y2": 131}, # real table rule + ]}} + assert [r["y1"] for r in horizontal_rules(page)] == [131] + + +def test_figure_refs_are_rewritten_to_the_figures_directory(tmp_path): + # liteparse emits bare filenames, which only resolve beside the markdown; figures are + # grouped in their own folder, so refs must be repointed or they break. + from src.parse_fulltext import rewrite_figure_refs + + md = tmp_path / "paper.md" + md.write_text("text ![](img_p4_1.png) more\n", encoding="utf-8") + rewrite_figure_refs(md, "../figures/paper") + assert "![](../figures/paper/img_p4_1.png)" in md.read_text(encoding="utf-8") diff --git a/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_parse_sections.py b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_parse_sections.py new file mode 100644 index 0000000..498d604 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/corpus-creation/tests/test_parse_sections.py @@ -0,0 +1,74 @@ +from pathlib import Path + +from src.parse_sections import ( + cross_check_dblp, + normalize_title, + parse_ceur_volume, + parse_dblp_titles, +) + +FIXTURES_DIR = Path(__file__).resolve().parent / "fixtures" + + +def test_parse_ceur_volume_extracts_sections_and_papers(): + raw_html = (FIXTURES_DIR / "ceur_vol_sample.html").read_text(encoding="utf-8") + sections = parse_ceur_volume(raw_html, "3497") + + assert len(sections) == 2 + assert sections[0]["lab_name"] == "Large-scale biomedical semantic indexing and question answering (BioASQ)" + assert len(sections[0]["papers"]) == 18 + + first_paper = sections[0]["papers"][0] + assert first_paper["title"] == "Overview of MedProcNER Task on Medical Procedure Detection and Entity Linking at BioASQ 2023" + assert first_paper["authors"][0] == "Salvador Lima-López" + assert first_paper["pdf_url"] == "https://ceur-ws.org/Vol-3497/paper-002.pdf" + assert first_paper["position_in_section"] == 1 + + +def test_parse_ceur_volume_normalizes_multiline_section_heading(): + raw_html = (FIXTURES_DIR / "ceur_vol_sample.html").read_text(encoding="utf-8") + sections = parse_ceur_volume(raw_html, "3497") + + checkthat_section = sections[1] + assert "\n" not in checkthat_section["lab_name"] + assert checkthat_section["lab_name"].startswith("Check-Worthiness, Subjectivity") + + +def test_normalize_title_treats_punctuation_as_word_boundary(): + # Real-world case: DBLP spells "CheckThat!-2023", CEUR spells "CheckThat! 2023" — + # the hyphen must not silently merge the two words into one token. + assert normalize_title("CheckThat!-2023") == normalize_title("CheckThat! 2023") + + +def test_parse_dblp_titles_builds_normalized_lookup(): + raw_html = (FIXTURES_DIR / "dblp_sample.html").read_text(encoding="utf-8") + lookup = parse_dblp_titles(raw_html) + + assert len(lookup) == 3 + key = normalize_title("Overview of MedProcNER Task on Medical Procedure Detection and Entity Linking at BioASQ 2023") + assert lookup[key] == ["Salvador Lima-Lopez", "Eulalia Farre-Maduell", "Luis Gasco"] + + +def test_cross_check_dblp_matches_and_flags_unmatched(caplog): + raw_ceur = (FIXTURES_DIR / "ceur_vol_sample.html").read_text(encoding="utf-8") + raw_dblp = (FIXTURES_DIR / "dblp_sample.html").read_text(encoding="utf-8") + sections = parse_ceur_volume(raw_ceur, "3497") + dblp_lookup = parse_dblp_titles(raw_dblp) + + import logging + logger = logging.getLogger("test") + cross_check_dblp(sections, dblp_lookup, logger) + + all_papers = [p for s in sections for p in s["papers"]] + matched = [p for p in all_papers if p["dblp_match"]] + unmatched = [p for p in all_papers if not p["dblp_match"]] + + # Only the 3 papers present in the small DBLP fixture should match; the rest + # of the 49-paper CEUR fixture is expected to be unmatched by design. + assert len(matched) == 3 + assert len(unmatched) == len(all_papers) - 3 + + # DBLP-preferred author spelling should replace the CEUR-parsed one on a match. + checkthat_match = next(p for p in all_papers if p["title"].startswith("CSECU-DSG")) + assert checkthat_match["dblp_match"] is True + assert checkthat_match["authors"] == ["Abdul Aziz"] diff --git a/conf27/inductive-nlg-for-shared-tasks/evaluation/Dockerfile b/conf27/inductive-nlg-for-shared-tasks/evaluation/Dockerfile new file mode 100644 index 0000000..da9eab4 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/evaluation/Dockerfile @@ -0,0 +1,14 @@ +# docker build -t mam10eks/uniagent-inlg:evaluator-0.0.1 . +FROM python:3-slim + +ADD requirements.txt / + +RUN pip3 install --no-cache-dir -r /requirements.txt \ + && pip3 --no-cache-dir install pytest \ + && rm /requirements.txt + +ADD *.py / + +RUN cd / \ + && PYTHONPATH=/ pytest test_evaluate.py + diff --git a/conf27/inductive-nlg-for-shared-tasks/evaluation/README.md b/conf27/inductive-nlg-for-shared-tasks/evaluation/README.md new file mode 100644 index 0000000..35ab68a --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/evaluation/README.md @@ -0,0 +1,14 @@ +# Summary Evaluation + +The evaluator matches predictions and references by `id` and reports BLEU, +chrF, ROUGE-1, ROUGE-2, and ROUGE-L. + +```bash +python3 evaluate.py \ + --predictions ../baseline-naive/foo.jsonl \ + --truths ../corpora/touche-20-task-1-spot-check/manual-paper-summaries.jsonl \ + --results results +``` + +The command writes `results/evaluation.prototext`. BLEU and chrF use a +0–100 scale; ROUGE values are mean per-document F-scores on a 0–1 scale. diff --git a/conf27/inductive-nlg-for-shared-tasks/evaluation/evaluate.py b/conf27/inductive-nlg-for-shared-tasks/evaluation/evaluate.py new file mode 100755 index 0000000..329b0be --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/evaluation/evaluate.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 + +import json +from pathlib import Path + +import click +import sacrebleu +from rouge_score import rouge_scorer + + +def read_summaries(path: Path) -> dict[str, str]: + summaries: dict[str, str] = {} + + with path.open(encoding="utf-8") as input_file: + for line_number, line in enumerate(input_file, start=1): + try: + record = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError( + f"{path}:{line_number}: invalid JSON: {error.msg}" + ) from error + + identifier = record.get("id") + summary = record.get("summary") + if not isinstance(identifier, str) or not identifier: + raise ValueError( + f"{path}:{line_number}: 'id' must be a non-empty string" + ) + if not isinstance(summary, str) or not summary.strip(): + raise ValueError( + f"{path}:{line_number}: 'summary' must be a non-empty string" + ) + if identifier in summaries: + raise ValueError(f"{path}:{line_number}: duplicate id '{identifier}'") + + summaries[identifier] = summary + + if not summaries: + raise ValueError(f"{path}: no summaries found") + return summaries + + +def align_summaries( + predictions: dict[str, str], truths: dict[str, str] +) -> tuple[list[str], list[str]]: + missing = sorted(truths.keys() - predictions.keys()) + unexpected = sorted(predictions.keys() - truths.keys()) + if missing or unexpected: + details = [] + if missing: + details.append(f"missing prediction ids: {', '.join(missing)}") + if unexpected: + details.append(f"unexpected prediction ids: {', '.join(unexpected)}") + raise ValueError("; ".join(details)) + + identifiers = sorted(truths) + return ( + [predictions[identifier] for identifier in identifiers], + [truths[identifier] for identifier in identifiers], + ) + + +def calculate_metrics( + predictions: list[str], truths: list[str] +) -> dict[str, float]: + scorer = rouge_scorer.RougeScorer( + ["rouge1", "rouge2", "rougeL"], use_stemmer=True + ) + rouge_scores = [ + scorer.score(truth, prediction) + for prediction, truth in zip(predictions, truths, strict=True) + ] + + return { + "BLEU": sacrebleu.metrics.BLEU(effective_order=True) + .corpus_score(predictions, [truths]) + .score, + "chrF": sacrebleu.corpus_chrf(predictions, [truths]).score, + "ROUGE-1": sum(score["rouge1"].fmeasure for score in rouge_scores) + / len(rouge_scores), + "ROUGE-2": sum(score["rouge2"].fmeasure for score in rouge_scores) + / len(rouge_scores), + "ROUGE-L": sum(score["rougeL"].fmeasure for score in rouge_scores) + / len(rouge_scores), + } + + +def write_prototext(metrics: dict[str, float], results_directory: Path) -> Path: + results_directory.mkdir(parents=True, exist_ok=True) + output_path = results_directory / "evaluation.prototext" + blocks = [ + f'measure {{\n key: "{name}"\n value: "{value:.6f}"\n}}' + for name, value in metrics.items() + ] + output_path.write_text("\n".join(blocks) + "\n", encoding="utf-8") + return output_path + + +@click.command() +@click.option( + "--predictions", + required=True, + type=click.Path(exists=True, dir_okay=False, path_type=Path), + help="Prediction JSONL with 'id' and 'summary' fields.", +) +@click.option( + "--truths", + required=True, + type=click.Path(exists=True, dir_okay=False, path_type=Path), + help="Ground-truth JSONL with 'id' and 'summary' fields.", +) +@click.option( + "--results", + "results_directory", + required=True, + type=click.Path(file_okay=False, path_type=Path), + help="Directory receiving evaluation.prototext.", +) +def main(predictions: Path, truths: Path, results_directory: Path) -> None: + try: + prediction_summaries = read_summaries(predictions) + truth_summaries = read_summaries(truths) + aligned_predictions, aligned_truths = align_summaries( + prediction_summaries, truth_summaries + ) + metrics = calculate_metrics(aligned_predictions, aligned_truths) + output_path = write_prototext(metrics, results_directory) + except ValueError as error: + raise click.ClickException(str(error)) from error + + click.echo(f"Wrote {output_path}") + + +if __name__ == "__main__": + main() diff --git a/conf27/inductive-nlg-for-shared-tasks/evaluation/requirements.txt b/conf27/inductive-nlg-for-shared-tasks/evaluation/requirements.txt new file mode 100644 index 0000000..aa24932 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/evaluation/requirements.txt @@ -0,0 +1,3 @@ +click +rouge-score +sacrebleu diff --git a/conf27/inductive-nlg-for-shared-tasks/evaluation/test_evaluate.py b/conf27/inductive-nlg-for-shared-tasks/evaluation/test_evaluate.py new file mode 100644 index 0000000..c7b9068 --- /dev/null +++ b/conf27/inductive-nlg-for-shared-tasks/evaluation/test_evaluate.py @@ -0,0 +1,83 @@ +import json + +import pytest +from click.testing import CliRunner + +from evaluate import ( + align_summaries, + calculate_metrics, + main, + read_summaries, + write_prototext, +) + + +def write_jsonl(path, records): + path.write_text( + "".join(json.dumps(record) + "\n" for record in records), + encoding="utf-8", + ) + + +def test_read_and_align_summaries(tmp_path): + predictions_path = tmp_path / "predictions.jsonl" + write_jsonl( + predictions_path, + [{"id": "2", "summary": "second"}, {"id": "1", "summary": "first"}], + ) + + predictions = read_summaries(predictions_path) + predicted, truths = align_summaries( + predictions, {"1": "reference one", "2": "reference two"} + ) + + assert predicted == ["first", "second"] + assert truths == ["reference one", "reference two"] + + +def test_rejects_mismatched_ids(): + with pytest.raises(ValueError, match="missing prediction ids: 2"): + align_summaries({"1": "prediction"}, {"1": "truth", "2": "truth"}) + + +def test_identical_summaries_receive_perfect_scores(): + metrics = calculate_metrics(["a complete summary"], ["a complete summary"]) + + assert metrics["BLEU"] == pytest.approx(100.0) + assert metrics["chrF"] == pytest.approx(100.0) + assert metrics["ROUGE-1"] == pytest.approx(1.0) + assert metrics["ROUGE-2"] == pytest.approx(1.0) + assert metrics["ROUGE-L"] == pytest.approx(1.0) + + +def test_write_prototext(tmp_path): + output_path = write_prototext({"BLEU": 12.3456789}, tmp_path) + + assert output_path.name == "evaluation.prototext" + assert output_path.read_text(encoding="utf-8") == ( + 'measure {\n key: "BLEU"\n value: "12.345679"\n}\n' + ) + + +def test_cli_writes_results(tmp_path): + predictions = tmp_path / "predictions.jsonl" + truths = tmp_path / "truths.jsonl" + results = tmp_path / "results" + records = [{"id": "1", "summary": "the same summary"}] + write_jsonl(predictions, records) + write_jsonl(truths, records) + + result = CliRunner().invoke( + main, + [ + "--predictions", + str(predictions), + "--truths", + str(truths), + "--results", + str(results), + ], + ) + + assert result.exit_code == 0 + assert (results / "evaluation.prototext").is_file()