lint.yml

name: Lint

on:
  push:
    branches:
      - master
  pull_request:

jobs:
  quick-checks:
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Checkout PyTorch
        uses: actions/checkout@v2
      - name: Install requirements
        id: requirements
        run: pip install -r requirements.txt
      - name: Ensure consistent CircleCI YAML config
        if: always() && steps.requirements.outcome == 'success'
        run: cd .circleci && ./ensure-consistency.py
      - name: Lint native_functions.yaml
        if: always() && steps.requirements.outcome == 'success'
        run: |
          pip install ruamel.yaml==0.17.4
          .github/scripts/lint_native_functions.py
      - name: Ensure correct trailing newlines
        if: always() && steps.requirements.outcome == 'success'
        run: |
          (! git --no-pager grep -Il '' -- . ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude)**.expect' ':(exclude)tools/clang_format_hash' | tools/trailing_newlines.py || (echo "The above files do not have correct trailing newlines; please normalize them"; false))
      - name: Ensure no trailing spaces
        if: always()
        run: |
          (! git --no-pager grep -In '[[:blank:]]$' -- . ':(exclude)**/contrib/**' ':(exclude)third_party' || (echo "The above lines have trailing spaces; please remove them"; false))
      - name: Ensure no tabs
        if: always()
        run: |
          (! git --no-pager grep -In $'\t' -- . ':(exclude)*.svg' ':(exclude)**Makefile' ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude).gitattributes' ':(exclude).gitmodules' || (echo "The above lines have tabs; please convert them to spaces"; false))
      - name: Ensure no non-breaking spaces
        if: always()
        run: |
          # NB: We use 'printf' below rather than '\u000a' since bash pre-4.2
          # does not support the '\u000a' syntax (which is relevant for local linters)
          (! git --no-pager grep -In "$(printf '\xC2\xA0')" -- . || (echo "The above lines have non-breaking spaces (U+00A0); please convert them to spaces (U+0020)"; false))
      - name: Ensure canonical include
        if: always()
        run: |
          (! git --no-pager grep -In $'#include "' -- ./c10 ./aten ./torch/csrc ':(exclude)aten/src/ATen/native/quantized/cpu/qnnpack/**' || (echo "The above lines have include with quotes; please convert them to #include <xxxx>"; false))
      - name: Ensure no versionless Python shebangs
        if: always()
        run: |
          (! git --no-pager grep -In '#!.*python$' -- . || (echo "The above lines have versionless Python shebangs; please specify either python2 or python3"; false))
      - name: Ensure no unqualified noqa
        if: always()
        run: |
          # shellcheck disable=SC2016
          (! git --no-pager grep -InP '# noqa(?!: [A-Z]+\d{3})' -- '**.py' '**.pyi' ':(exclude)caffe2' || (echo 'The above lines have unqualified `noqa`; please convert them to `noqa: XXXX`'; false))
      - name: Ensure no unqualified type ignore
        if: always()
        run: |
          # shellcheck disable=SC2016
          (! git --no-pager grep -InP '# type:\s*ignore(?!\[)' -- '**.py' '**.pyi' ':(exclude)test/test_jit.py' || (echo 'The above lines have unqualified `type: ignore`; please convert them to `type: ignore[xxxx]`'; false))
      # note that this next step depends on a clean checkout;
      # if you run it locally then it will likely to complain
      # about all the generated files in torch/test
      - name: Ensure C++ source files are not executable
        if: always()
        run: |
          # shellcheck disable=SC2016
          (! find . \( -path ./third_party -o -path ./.git -o -path ./torch/bin -o -path ./build \) -prune -o -type f -executable -regextype posix-egrep -not -regex '.+(\.(bash|sh|py|so)|git-pre-commit|git-clang-format|gradlew)$' -print | grep . || (echo 'The above files have executable permission; please remove their executable permission by using `chmod -x`'; false))
      - name: C++ docs check
        if: always() && steps.requirements.outcome == 'success'
        run: |
          sudo apt-get install -y doxygen
          cd docs/cpp/source && ./check-doxygen.sh
      - name: CUDA kernel launch check
        if: always() && steps.requirements.outcome == 'success'
        run: |
          set -eux
          python torch/testing/_check_kernel_launches.py |& tee "${GITHUB_WORKSPACE}"/cuda_kernel_launch_checks.txt
      - name: Ensure no direct cub include
        if: always()
        run: |
          (! git --no-pager grep -I -no $'#include <cub/' --  ./aten  ':(exclude)aten/src/ATen/cuda/cub.cuh' || (echo "The above files have direct cub include; please include ATen/cuda/cub.cuh instead and wrap your cub calls in at::native namespace if necessary"; false))

  py2-setup-validate-errormsg:
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 2.x
          architecture: x64
      - name: Checkout PyTorch
        uses: actions/checkout@v2
      - name: Attempt to run setup.py
        run: |
          python2 setup.py | grep "Python 2 has reached end-of-life and is no longer supported by PyTorch."

  shellcheck:
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Checkout PyTorch
        uses: actions/checkout@v2
      - name: Install requirements
        id: requirements
        run: |
          pip install -r requirements.txt
      - name: Install Jinja2
        run: |
          pip install Jinja2==3.0.1
      - name: Checkout PyTorch
        uses: actions/checkout@v2
      - name: Regenerate workflows
        id: generate_workflows
        run: .github/scripts/generate_ci_workflows.py
      - name: Assert that regenerating the workflows didn't change them
        run: .github/scripts/report_git_status.sh .github/workflows
      - name: Install ShellCheck
        id: install_shellcheck
        if: always()
        # https://github.com/koalaman/shellcheck/tree/v0.7.2#installing-a-pre-compiled-binary
        run: |
          set -x
          scversion="v0.7.2"
          wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${scversion?}/shellcheck-${scversion?}.linux.x86_64.tar.xz" | tar -xJv
          sudo cp "shellcheck-${scversion}/shellcheck" /usr/bin/
          rm -r "shellcheck-${scversion}"
          shellcheck --version
      - name: Extract scripts from GitHub Actions workflows
        if: always() && steps.install_shellcheck.outcome == 'success'
        run: |
          # For local lints, remove the .extracted_scripts folder if it was already there
          rm -rf .extracted_scripts
          tools/extract_scripts.py --out=.extracted_scripts
      - name: Run ShellCheck
        if: always() && steps.install_shellcheck.outcome == 'success'
        run: |
          tools/run_shellcheck.sh .extracted_scripts .jenkins/pytorch
      - name: Check that jobs will be cancelled
        if: always() && steps.generate_workflows.outcome == 'success'
        run: |
          .github/scripts/ensure_actions_will_cancel.py

  toc:
    runs-on: ubuntu-18.04
    # https://github.com/actions/virtual-environments/issues/599#issuecomment-602754687
    env:
      NPM_CONFIG_PREFIX: ~/.npm-global
    steps:
      - name: Setup Node
        uses: actions/setup-node@v2
      - name: Checkout PyTorch
        uses: actions/checkout@v2
      - name: Install markdown-toc
        run: npm install -g markdown-toc
      - name: Regenerate ToCs and check that they didn't change
        run: |
          set -eux
          export PATH=~/.npm-global/bin:"$PATH"
          for FILE in $(git grep -Il '<!-- toc -->' -- '**.md'); do
            markdown-toc --bullets='-' -i "$FILE"
          done

          .github/scripts/report_git_status.sh

  flake8-py3:
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Fetch PyTorch
        uses: actions/checkout@v2
        with:
          fetch-depth: 2 # to allow us to use github.event.pull_request.head.sha
      - name: Prepare output dir with HEAD commit SHA
        env:
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          mkdir flake8-output
          cd flake8-output
          echo "$HEAD_SHA" > commit-sha.txt
      - name: Install dependencies
        run: |
          set -eux
          pip install typing-extensions # for tools/translate_annotations.py
          pip install -r requirements-flake8.txt
          flake8 --version
      - name: Run flake8
        run: |
          set -eux
          flake8 | tee "${GITHUB_WORKSPACE}"/flake8-output.txt
      - name: Translate annotations
        if: github.event_name == 'pull_request'
        env:
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          tools/translate_annotations.py \
            --file="${GITHUB_WORKSPACE}"/flake8-output.txt \
            --regex='^(?P<filename>.*?):(?P<lineNumber>\d+):(?P<columnNumber>\d+): (?P<errorCode>\w+\d+) (?P<errorDesc>.*)' \
            --commit="$HEAD_SHA" \
            > flake8-output/annotations.json
      - name: Upload artifact
        uses: actions/upload-artifact@v2
        with:
          name: flake8-py3
          path: flake8-output/
      - name: Fail if there were any warnings
        run: |
          set -eux
          # Re-output flake8 status so GitHub logs show it on the step that actually failed
          cat "${GITHUB_WORKSPACE}"/flake8-output.txt
          [ ! -s "${GITHUB_WORKSPACE}"/flake8-output.txt ]

  clang-tidy:
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-18.04
    container:
      # ubuntu18.04-cuda10.2-py3.6-tidy11
      image: ghcr.io/pytorch/cilint-clang-tidy:e2cfc57ce4fa3a257a4b78fdfdc2b065c167b9c5
    steps:
      - name: Checkout PyTorch
        uses: actions/checkout@v2
        with:
          fetch-depth: 0 # to allow tools/clang_tidy.py to do its thing
      - name: Prepare output dir with HEAD commit SHA
        env:
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          cd "${GITHUB_WORKSPACE}"
          mkdir clang-tidy-output
          cd clang-tidy-output
          echo "$HEAD_SHA" > commit-sha.txt
      - name: Generate build files
        run: |
          cd "${GITHUB_WORKSPACE}"
          set -eux
          git remote add upstream https://github.com/pytorch/pytorch
          git fetch upstream "$GITHUB_BASE_REF"

          if [ ! -d build ]; then
            git submodule update --init --recursive

            export USE_NCCL=0
            export USE_DEPLOY=1
            # We really only need compile_commands.json, so no need to build!
            time python3 setup.py --cmake-only build

            # Generate ATen files.
            time python3 -m tools.codegen.gen \
              -s aten/src/ATen \
              -d build/aten/src/ATen

            # Generate PyTorch files.
            time python3 tools/setup_helpers/generate_code.py            \
              --declarations-path build/aten/src/ATen/Declarations.yaml \
              --native-functions-path aten/src/ATen/native/native_functions.yaml \
              --nn-path aten/src
          fi
      - name: Run clang-tidy
        env:
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          cd "${GITHUB_WORKSPACE}"
          set -eux

          # Run Clang-Tidy
          # The negative filters below are to exclude files that include onnx_pb.h or
          # caffe2_pb.h, otherwise we'd have to build protos as part of this CI job.
          # FunctionsManual.cpp is excluded to keep this diff clean. It will be fixed
          # in a follow up PR.
          # /torch/csrc/generic/*.cpp is excluded because those files aren't actually built.
          # deploy/interpreter files are excluded due to using macros and other techniquies
          # that are not easily converted to accepted c++
          python3 tools/clang_tidy.py                               \
            --verbose                                              \
            --paths torch/csrc/                                    \
            --diff "$BASE_SHA"                                   \
            -g"-torch/csrc/jit/passes/onnx/helper.cpp"             \
            -g"-torch/csrc/jit/passes/onnx/shape_type_inference.cpp"\
            -g"-torch/csrc/jit/serialization/onnx.cpp"             \
            -g"-torch/csrc/jit/serialization/export.cpp"           \
            -g"-torch/csrc/jit/serialization/import.cpp"           \
            -g"-torch/csrc/jit/serialization/import_legacy.cpp"    \
            -g"-torch/csrc/onnx/init.cpp"                          \
            -g"-torch/csrc/cuda/nccl.*"                            \
            -g"-torch/csrc/cuda/python_nccl.cpp"                   \
            -g"-torch/csrc/autograd/FunctionsManual.cpp"           \
            -g"-torch/csrc/generic/*.cpp"                          \
            -g"-torch/csrc/jit/codegen/cuda/runtime/*"             \
            -g"-torch/csrc/deploy/interpreter/interpreter.cpp"     \
            -g"-torch/csrc/deploy/interpreter/interpreter.h"  \
            -g"-torch/csrc/deploy/interpreter/interpreter_impl.h"  \
            -g"-torch/csrc/deploy/interpreter/test_main.cpp"  \
            "$@" > "${GITHUB_WORKSPACE}"/clang-tidy-output.txt

          cat "${GITHUB_WORKSPACE}"/clang-tidy-output.txt

          tools/translate_annotations.py \
            --file=clang-tidy-output.txt \
            --regex='^(?P<filename>.*?):(?P<lineNumber>\d+):(?P<columnNumber>\d+): (?P<errorDesc>.*?) \[(?P<errorCode>.*)\]' \
            --commit="$HEAD_SHA" \
            > clang-tidy-output/annotations.json
      - name: Upload artifact
        uses: actions/upload-artifact@v2
        with:
          name: clang-tidy
          path: clang-tidy-output/

  cmakelint:
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Fetch PyTorch
        uses: actions/checkout@v2
      - name: Install dependencies
        run: |
          set -eux
          pip install cmakelint
          cmakelint --version
      - name: Run cmakelint
        run: |
          set -eux
          git ls-files -z -- bootstrap '*.cmake' '*.cmake.in' '*CMakeLists.txt' | \
          grep -E -z -v '^(cmake/Modules/|cmake/Modules_CUDA_fix/|cmake/Caffe2Config.cmake.in|aten/src/ATen/ATenConfig.cmake.in|cmake/Caffe2ConfigVersion.cmake.in|cmake/TorchConfig.cmake.in|cmake/TorchConfigVersion.cmake.in|cmake/cmake_uninstall.cmake.in)' | \
          xargs -0 cmakelint --config=.cmakelintrc --spaces=2 --quiet

  mypy:
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
          architecture: x64
      - name: Fetch PyTorch
        uses: actions/checkout@v2
      - name: Install dependencies
        run: |
          set -eux
          pip install -r requirements.txt
          pip install mypy==0.812
          # Needed to check tools/render_junit.py
          pip install junitparser rich
      - name: Run autogen
        run: |
          set -eux
          time python -mtools.generate_torch_version --is_debug=false
          time python -mtools.codegen.gen -s aten/src/ATen -d build/aten/src/ATen
          time python -mtools.pyi.gen_pyi --native-functions-path aten/src/ATen/native/native_functions.yaml --deprecated-functions-path "tools/autograd/deprecated.yaml"
      - name: Run mypy
        run: |
          set -eux
          for CONFIG in mypy*.ini; do mypy --config="$CONFIG"; done

concurrency:
  group: lint-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true