diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1f39ac1..19ca3c8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -51,7 +51,7 @@ jobs: uses: ./.github/workflows/reusable-build.yml with: CIBW_SKIP: "pp* cp36-* cp37-*" - CIBW_BUILD: "cp*-macosx* cp*-manylinux*" + CIBW_BUILD: "cp*-macosx* cp*-manylinux* cp*-win*" VERSION: ${{ github.ref_name }} secrets: inherit diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml index abb00a8..a86d222 100644 --- a/.github/workflows/reusable-build.yml +++ b/.github/workflows/reusable-build.yml @@ -17,7 +17,7 @@ jobs: build-wheels: strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-latest] fail-fast: false name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aa36d11..d36d1e2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,7 +41,7 @@ jobs: with: python-version: '3.12' cache: 'pip' # caching pip dependencies - - name: Install dependencies + - name: Install lib from source and dependencies run: | python -m pip install -e .[test] - name: Run tests @@ -52,7 +52,7 @@ jobs: uses: ./.github/workflows/reusable-build.yml with: CIBW_SKIP: "pp* cp36-* cp37-*" - CIBW_BUILD: "cp*-macosx* cp*-manylinux*" + CIBW_BUILD: "cp*-macosx* cp*-manylinux* cp*-win*" secrets: inherit full-tests-python: @@ -60,7 +60,7 @@ jobs: strategy: matrix: python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-latest] fail-fast: false name: Test wheel on ${{ matrix.os }} and Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} @@ -72,16 +72,21 @@ jobs: path: dist - name: Show dist files run: ls -lah ./dist + shell: bash - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} cache: 'pip' # caching pip dependencies - - name: Install dependencies + - name: Remove sdist package to force install wheel later + run: | + rm -rf ./dist/*.tar.gz + shell: bash + - name: Install lib and dependencies run: | # force install package from local dist directory + pip uninstall -y codebleu || true # TODO: check the sdist package is not installed - rm -rf ./dist/*.tar.gz pip install --upgrade --no-deps --no-index --find-links=./dist codebleu # install dependencies for the package and tests pip install .[test] diff --git a/.gitignore b/.gitignore index 4baa45b..6d24843 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,15 @@ codebleu/parser/*.so codebleu/parser/*.dylib codebleu/parser/*.dll +/codebleu/parser/tree-sitter-c-sharp/ +/codebleu/parser/tree-sitter-go/ +/codebleu/parser/tree-sitter-java/ +/codebleu/parser/tree-sitter-javascript/ +/codebleu/parser/tree-sitter-php/ +/codebleu/parser/tree-sitter-python/ +/codebleu/parser/tree-sitter-ruby/ +/tree_sitter/ +codebleu/*.so # Byte-compiled / optimized / DLL files @@ -166,10 +175,3 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ -/codebleu/parser/tree-sitter-c-sharp/ -/codebleu/parser/tree-sitter-go/ -/codebleu/parser/tree-sitter-java/ -/codebleu/parser/tree-sitter-javascript/ -/codebleu/parser/tree-sitter-php/ -/codebleu/parser/tree-sitter-python/ -/codebleu/parser/tree-sitter-ruby/ diff --git a/README.md b/README.md index 9827fcc..510e310 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![PyPI version](https://badge.fury.io/py/codebleu.svg)](https://badge.fury.io/py/codebleu) -This repository contains an unofficial `CodeBLEU` implementation that supports Linux and MacOS. It is available through `PyPI` and the `evaluate` library. +This repository contains an unofficial `CodeBLEU` implementation that supports `Linux`, `MacOS` and `Windows`. It is available through `PyPI` and the `evaluate` library. The code is based on the original [CodeXGLUE/CodeBLEU](https://github.com/microsoft/CodeXGLUE/tree/main/Code-Code/code-to-code-trans/evaluator/CodeBLEU) and updated version by [XLCoST/CodeBLEU](https://github.com/reddy-lab-code-research/XLCoST/tree/main/code/translation/evaluator/CodeBLEU). It has been refactored, tested, built for macOS, and multiple improvements have been made to enhance usability @@ -28,7 +28,7 @@ The metric has shown higher correlation with human evaluation than `BLEU` and `a ## Installation As this library require `so` file compilation it is platform dependent. -Currently available for `Linux` (manylinux) and `MacOS` with Python 3.8+. +Currently available for `Linux` (manylinux), `MacOS` and `Windows` with Python 3.8+. The metrics is available as [pip package](https://pypi.org/project/codebleu/) and can be installed as indicated above: ```bash diff --git a/codebleu/codebleu.py b/codebleu/codebleu.py index 1319edd..2e9adbf 100644 --- a/codebleu/codebleu.py +++ b/codebleu/codebleu.py @@ -17,7 +17,7 @@ def calc_codebleu( weights: Tuple[float, float, float, float] = (0.25, 0.25, 0.25, 0.25), tokenizer: Optional[Callable] = None, keywords_dir: Path = PACKAGE_DIR / "keywords", - lang_so_file: Path = PACKAGE_DIR / "parser" / "my-languages.so", + lang_so_file: Path = PACKAGE_DIR / "my-languages.so", ) -> Dict[str, float]: """Calculate CodeBLEU score @@ -69,10 +69,10 @@ def make_weights(reference_tokens, key_word_list): weighted_ngram_match_score = weighted_ngram_match.corpus_bleu(tokenized_refs_with_weights, tokenized_hyps) # calculate syntax match - syntax_match_score = syntax_match.corpus_syntax_match(references, hypothesis, lang, lang_so_file) + syntax_match_score = syntax_match.corpus_syntax_match(references, hypothesis, lang, str(lang_so_file)) # calculate dataflow match - dataflow_match_score = dataflow_match.corpus_dataflow_match(references, hypothesis, lang, lang_so_file) + dataflow_match_score = dataflow_match.corpus_dataflow_match(references, hypothesis, lang, str(lang_so_file)) alpha, beta, gamma, theta = weights code_bleu_score = ( diff --git a/codebleu/parser/build.py b/codebleu/parser/build.py deleted file mode 100644 index 3e409e2..0000000 --- a/codebleu/parser/build.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Copyright (c) 2023 Konstantin Chernyshev. -# Licensed under the MIT license. -from tree_sitter import Language - -if __name__ == "__main__": - Language.build_library( - "my-languages.so", - [ - "tree-sitter/go", - "tree-sitter/javascript", - "tree-sitter/python", - "tree-sitter/php", - "tree-sitter/java", - "tree-sitter/ruby", - "tree-sitter/c-sharp", - "tree-sitter/c", - "tree-sitter/cpp", - ], - ) diff --git a/codebleu/parser/build.sh b/codebleu/parser/build.sh deleted file mode 100644 index 0189561..0000000 --- a/codebleu/parser/build.sh +++ /dev/null @@ -1,11 +0,0 @@ -git clone --depth=1 --single-branch https://github.com/tree-sitter/tree-sitter-go tree-sitter/go -git clone --depth=1 --single-branch https://github.com/tree-sitter/tree-sitter-javascript tree-sitter/javascript -git clone --depth=1 --single-branch https://github.com/tree-sitter/tree-sitter-python tree-sitter/python -git clone --depth=1 --single-branch https://github.com/tree-sitter/tree-sitter-ruby tree-sitter/ruby -git clone --depth=1 --single-branch https://github.com/tree-sitter/tree-sitter-php tree-sitter/php -git clone --depth=1 --single-branch https://github.com/tree-sitter/tree-sitter-java tree-sitter/java -git clone --depth=1 --single-branch https://github.com/tree-sitter/tree-sitter-c-sharp tree-sitter/c-sharp -git clone --depth=1 --single-branch https://github.com/tree-sitter/tree-sitter-c tree-sitter/c -git clone --depth=1 --single-branch https://github.com/tree-sitter/tree-sitter-cpp tree-sitter/cpp -python build.py -#rm -rf tree-sitter diff --git a/pyproject.toml b/pyproject.toml index 22d561c..c2b9f78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,17 +1,17 @@ [build-system] -requires = ["setuptools>=61.0.0", "wheel", "tree-sitter>=0.20.0,<1.0.0"] +requires = ["setuptools>=61.0.0", "wheel", "tree-sitter>=0.20.0,<1.0.0", "requests>=2.0.0,<3.0.0"] build-backend = "setuptools.build_meta" [project] name = "codebleu" -description = "Unofficial CodeBLEU implementation that supports Linux and MacOS available on PyPI." +description = "Unofficial CodeBLEU implementation that supports Linux, MacOS and Windows available on PyPI." readme = "README.md" license = {text = "MIT License"} authors = [ {name = "Konstantin Chernyshev", email = "kdchernyshev@gmail.com"}, ] -keywords = ["codebleu", "code", "bleu", "nlp", "natural language processing", "programming", "evaluate", "evaluation", "code generation", "matrics"] +keywords = ["codebleu", "code", "bleu", "nlp", "natural language processing", "programming", "evaluate", "evaluation", "code generation", "metrics"] dynamic = ["version"] requires-python = ">=3.8" @@ -77,7 +77,7 @@ warn_redundant_casts = true warn_unused_ignores = true warn_unreachable = true allow_untyped_decorators = true -exclude = ["codebleu/parser/tree-sitter", "codebleu/parser/tree-sitter/python"] +exclude = ["codebleu/parser/tree-sitter", "codebleu/parser/tree-sitter/python", "tree_sitter"] [tool.pytest.ini_options] minversion = "6.0" @@ -86,7 +86,7 @@ python_files = "test_*.py" addopts = "--cov=codebleu/ --cov-report term-missing" [tool.coverage.run] -omit = ["tests/*", "codebleu/parser/tree-sitter/*"] +omit = ["tests/*", "codebleu/parser/tree-sitter/*", "tree_sitter"] [tool.isort] @@ -95,7 +95,7 @@ src_paths = ["codebleu", "tests"] known_first_party = ["codebleu", "tests"] line_length = 120 combine_as_imports = true -skip = ["build", "dist", ".venv", ".eggs", ".mypy_cache", ".pytest_cache", ".git", ".tox", ".nox", "codebleu/parser"] +skip = ["build", "dist", ".venv", ".eggs", ".mypy_cache", ".pytest_cache", ".git", ".tox", ".nox", "codebleu/parser", "tree_sitter"] [tool.black] line_length=120 diff --git a/setup.py b/setup.py index 1c12d40..b30620c 100644 --- a/setup.py +++ b/setup.py @@ -1,19 +1,80 @@ -import subprocess +from __future__ import annotations + +import io +import shutil +import zipfile from pathlib import Path +import requests from setuptools import setup from setuptools.dist import Distribution +from tree_sitter import Language + ROOT = Path(__file__).parent -subprocess.run( - ["bash", "build.sh"], - cwd=ROOT / "codebleu" / "parser", - check=True, +tree_sitter_languages = { + "go": "https://github.com/tree-sitter/tree-sitter-go/archive/refs/tags/v0.20.0.zip", + "javascript": "https://github.com/tree-sitter/tree-sitter-javascript/archive/refs/tags/v0.20.1.zip", + "python": "https://github.com/tree-sitter/tree-sitter-python/archive/refs/tags/v0.20.4.zip", + "ruby": "https://github.com/tree-sitter/tree-sitter-ruby/archive/refs/tags/v0.19.0.zip", + "php": "https://github.com/tree-sitter/tree-sitter-php/archive/refs/tags/v0.19.0.zip", + "java": "https://github.com/tree-sitter/tree-sitter-java/archive/refs/tags/v0.20.2.zip", + "c-sharp": "https://github.com/tree-sitter/tree-sitter-c-sharp/archive/refs/tags/v0.20.0.zip", + "c": "https://github.com/tree-sitter/tree-sitter-c/archive/refs/tags/v0.20.6.zip", + "cpp": "https://github.com/tree-sitter/tree-sitter-cpp/archive/refs/tags/v0.20.3.zip", +} + + +def download_tree_sitter_languages(languages: dict[str, str], languages_folder: Path) -> list[str]: + if languages_folder.exists(): + shutil.rmtree(languages_folder) + languages_folder.mkdir(parents=True) + + extracted_folders: list[str] = [] + for lang, url in languages.items(): + # Download the ZIP file + response = requests.get(url) + response.raise_for_status() + + # Extract the ZIP file + with zipfile.ZipFile(io.BytesIO(response.content)) as zip_f: + zip_f.extractall(languages_folder) + extracted_folders.append(zip_f.namelist()[0]) # get the name of the extracted folder + + return extracted_folders + + +def build_tree_sitter_languages(languages: dict[str, str], languages_folder: Path, target_lib_file: Path) -> str: + extracted_folders = download_tree_sitter_languages(languages, languages_folder) + + Language.build_library( + str(target_lib_file), + [str(languages_folder / lang_folder) for lang_folder in extracted_folders], + ) + + return str(target_lib_file) + + +build_tree_sitter_languages( + tree_sitter_languages, + ROOT / "tree_sitter", + ROOT / "codebleu" / "my-languages.so", ) +# tree_sitter_extension = Extension( +# 'codebleu.tree_sitter', +# sources=[], +# include_dirs=[], +# libraries=[], +# extra_objects=[ +# +# ], +# ) + + class PlatformSpecificDistribution(Distribution): """Distribution which always forces a binary package with platform name""" @@ -21,4 +82,6 @@ def has_ext_modules(self): return True -setup(distclass=PlatformSpecificDistribution) +setup( + distclass=PlatformSpecificDistribution, +) diff --git a/tests/test_codebleu.py b/tests/test_codebleu.py index 2c41935..35414be 100644 --- a/tests/test_codebleu.py +++ b/tests/test_codebleu.py @@ -65,8 +65,8 @@ def test_error_when_input_length_mismatch() -> None: ["public static int Sign ( double d ) { return ( float ) ( ( d == 0 ) ? 0 : ( c < 0.0 ) ? - 1 : 1) ; }"], ["public static int Sign ( double d ) { return ( int ) ( ( d == 0 ) ? 0 : ( d < 0 ) ? - 1 : 1) ; }"], 0.7846, - 11/19, # In example, it is 13/21, but with new version of tree-sitter it is 11/19 - 2/3, + 11 / 19, # In example, it is 13/21, but with new version of tree-sitter it is 11/19 + 2 / 3, 0.7019, # Should be 0.7238 if AST=13/21 in the paper, however at the moment tee-sitter AST is 11/19 ), # https://arxiv.org/pdf/2009.10297.pdf "3.4 Two Examples" at the page 4 @@ -74,8 +74,8 @@ def test_error_when_input_length_mismatch() -> None: ["public static int Sign ( double d ) { return ( float ) ( ( d == 0 ) ? 0 : ( c < 0.0 ) ? - 1 : 1) ;"], ["public static int Sign ( double d ) { return ( int ) ( ( d == 0 ) ? 0 : ( d < 0 ) ? - 1 : 1) ; }"], 0.7543, - 11/19, # In example, it is 13/21, but with new version of tree-sitter it is 11/19 - 2/3, + 11 / 19, # In example, it is 13/21, but with new version of tree-sitter it is 11/19 + 2 / 3, 0.6873, # Should be 0.6973 if AST=13/21 in the paper, however at the moment tee-sitter AST is 11/19 ), # https://arxiv.org/pdf/2009.10297.pdf "3.4 Two Examples" at the page 4