diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 7c54e22a35..b59346ca1a 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -32,13 +32,14 @@ If some part is not as extensible, you can also bring up the issue to make it mo When sending a PR, please do: -1. Fork the repo and create your branch from `master`. +1. If a PR contains multiple orthogonal changes, split it to several PRs. 2. If you've added code that should be tested, add tests. -3. If APIs are changed, update the documentation. -4. Ensure the test suite passes. -5. Make sure your code lints with `./dev/linter.sh`. -6. If a PR contains multiple orthogonal changes, split it to several PRs. -7. If you haven't already, complete the Contributor License Agreement ("CLA"). +3. For PRs that need experiments (e.g. adding a new model), you don't need to update model zoo, + but do provide experiment results in the description of the PR. +4. If APIs are changed, update the documentation. +5. Ensure the test suite passes. +6. Make sure your code lints with `./dev/linter.sh`. + ## Contributor License Agreement ("CLA") In order to accept your pull request, we need you to submit a CLA. You only need diff --git a/.gitignore b/.gitignore index c4241798e0..23b8ca5c8d 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ _ext detectron2.egg-info/ build/ dist/ +wheels/ # pytorch/python/numpy formats *.pth diff --git a/INSTALL.md b/INSTALL.md index ad27beaada..d8cb3b7ed7 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -12,12 +12,11 @@ also installs detectron2 with a few simple commands. You can install them together at [pytorch.org](https://pytorch.org) to make sure of this. - OpenCV, optional, needed by demo and visualization - pycocotools: `pip install cython; pip install 'git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI'` -- gcc & g++ ≥ 4.9 -### Build and Install Detectron2 +### Build Detectron2 from Source -After having the above dependencies, run: +After having the above dependencies and gcc & g++ ≥ 4.9, run: ``` pip install 'git+https://github.com/facebookresearch/detectron2.git' # (add --user if you don't have permission) @@ -33,8 +32,21 @@ cd detectron2 && pip install -e . To __rebuild__ detectron2 that's built from a local clone, `rm -rf build/ **/*.so` then `pip install -e .`. You often need to rebuild detectron2 after reinstalling PyTorch. +### Install Pre-Built Detectron2 +``` +# for CUDA 10.1: +pip install detectron2 -f \ + https://dl.fbaipublicfiles.com/detectron2/wheels/cu101/index.html +``` +You can replace cu101 with "cu{100,92}" or "cpu". + +Note that such installation has to be used with the latest official PyTorch release (currently 1.4). +It will not work with your custom build of PyTorch. + ### Common Installation Issues +If you met issues using the pre-built detectron2, please uninstall it and try building it from source. + Click each issue for its solutions: <details> diff --git a/dev/packaging/README.md b/dev/packaging/README.md new file mode 100644 index 0000000000..095684fcc1 --- /dev/null +++ b/dev/packaging/README.md @@ -0,0 +1,17 @@ + +## To build a cu101 wheel for release: + +``` +$ nvidia-docker run -it --storage-opt "size=20GB" --name pt pytorch/manylinux-cuda101 +# inside the container: +# git clone https://github.com/facebookresearch/detectron2/ +# cd detectron2 +# export CU_VERSION=cu101 D2_VERSION_SUFFIX= PYTHON_VERSION=3.7 PYTORCH_VERSION=1.4 +# ./dev/packaging/build_wheel.sh +``` + +## To build all wheels for `CUDA {9.2,10.0,10.1}` x `Python {3.6,3.7,3.8}`: +``` +./dev/packaging/build_all_wheels.sh +./dev/packaging/gen_wheel_index.sh /path/to/wheels +``` diff --git a/dev/packaging/build_all_wheels.sh b/dev/packaging/build_all_wheels.sh new file mode 100755 index 0000000000..99d9492c60 --- /dev/null +++ b/dev/packaging/build_all_wheels.sh @@ -0,0 +1,56 @@ +#!/bin/bash -e + +PYTORCH_VERSION=1.4 + +build_for_one_cuda() { + cu=$1 + + case "$cu" in + cu*) + container_name=manylinux-cuda${cu/cu/} + ;; + cpu) + container_name=manylinux-cuda101 + ;; + *) + echo "Unrecognized cu=$cu" + exit 1 + ;; + esac + + echo "Launching container $container_name ..." + + for py in 3.6 3.7 3.8; do + docker run -itd \ + --name $container_name \ + --mount type=bind,source="$(pwd)",target=/detectron2 \ + pytorch/$container_name + + cat <<EOF | docker exec -i $container_name sh + export CU_VERSION=$cu D2_VERSION_SUFFIX=+$cu PYTHON_VERSION=$py + export PYTORCH_VERSION=$PYTORCH_VERSION + cd /detectron2 && ./dev/packaging/build_wheel.sh +EOF + + if [[ "$cu" == "cu101" ]]; then + # build wheel without local version + cat <<EOF | docker exec -i $container_name sh + export CU_VERSION=$cu D2_VERSION_SUFFIX= PYTHON_VERSION=$py + export PYTORCH_VERSION=$PYTORCH_VERSION + cd /detectron2 && ./dev/packaging/build_wheel.sh +EOF + fi + + docker exec -i $container_name rm -rf /detectron2/build/$cu + docker container stop $container_name + docker container rm $container_name + done +} + +if [[ -n "$1" ]]; then + build_for_one_cuda "$1" +else + for cu in cu101 cu100 cu92 cpu; do + build_for_one_cuda "$cu" + done +fi diff --git a/dev/packaging/build_wheel.sh b/dev/packaging/build_wheel.sh new file mode 100755 index 0000000000..c336dcb087 --- /dev/null +++ b/dev/packaging/build_wheel.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -ex + +ldconfig # https://github.com/NVIDIA/nvidia-docker/issues/854 + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +. "$script_dir/pkg_helpers.bash" + +echo "Build Settings:" +echo "CU_VERSION: $CU_VERSION" # e.g. cu100 +echo "D2_VERSION_SUFFIX: $D2_VERSION_SUFFIX" # e.g. +cu100 or "" +echo "PYTHON_VERSION: $PYTHON_VERSION" # e.g. 3.6 +echo "PYTORCH_VERSION: $PYTORCH_VERSION" # e.g. 1.4 + +setup_cuda +setup_wheel_python + +export TORCH_VERSION_SUFFIX="+$CU_VERSION" +if [[ "$CU_VERSION" == "cu101" ]]; then + export TORCH_VERSION_SUFFIX="" +fi +pip_install pip numpy -U +pip_install "torch==$PYTORCH_VERSION$TORCH_VERSION_SUFFIX" \ + -f https://download.pytorch.org/whl/$CU_VERSION/torch_stable.html + +# use separate directories to allow parallel build +BASE_BUILD_DIR=build/$CU_VERSION/$PYTHON_VERSION +python setup.py \ + build -b $BASE_BUILD_DIR \ + bdist_wheel -b $BASE_BUILD_DIR/build_dist -d wheels/$CU_VERSION diff --git a/dev/packaging/gen_wheel_index.sh b/dev/packaging/gen_wheel_index.sh new file mode 100755 index 0000000000..10b497a734 --- /dev/null +++ b/dev/packaging/gen_wheel_index.sh @@ -0,0 +1,24 @@ +#!/bin/bash -e + + +root=$1 +if [[ -z "$root" ]]; then + echo "Usage: ./gen_wheel_index.sh /path/to/wheels" + exit +fi + +index=$root/index.html + +cd "$root" +for cu in cpu cu92 cu100 cu101; do + cd $cu + for whl in *.whl; do + echo "<a href=\"$whl\">$whl</a><br>" + done > index.html + cd "$root" +done + +for whl in $(find . -type f -name '*.whl' -printf '%P\n' | sort); do + echo "<a href=\"$whl\">$whl</a><br>" +done > "$index" + diff --git a/dev/packaging/pkg_helpers.bash b/dev/packaging/pkg_helpers.bash new file mode 100755 index 0000000000..a029091dc6 --- /dev/null +++ b/dev/packaging/pkg_helpers.bash @@ -0,0 +1,52 @@ +#!/bin/bash -e + +# Function to retry functions that sometimes timeout or have flaky failures +retry () { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) +} +# Install with pip a bit more robustly than the default +pip_install() { + retry pip install --progress-bar off "$@" +} + + +setup_cuda() { + # Now work out the CUDA settings + # Like other torch domain libraries, we choose common GPU architectures only. + export FORCE_CUDA=1 + case "$CU_VERSION" in + cu101) + export CUDA_HOME=/usr/local/cuda-10.1/ + export TORCH_CUDA_ARCH_LIST="3.5;3.7;5.0;5.2;6.0+PTX;6.1+PTX;7.0+PTX;7.5+PTX" + ;; + cu100) + export CUDA_HOME=/usr/local/cuda-10.0/ + export TORCH_CUDA_ARCH_LIST="3.5;3.7;5.0;5.2;6.0+PTX;6.1+PTX;7.0+PTX;7.5+PTX" + ;; + cu92) + export CUDA_HOME=/usr/local/cuda-9.2/ + export TORCH_CUDA_ARCH_LIST="3.5;3.7;5.0;5.2;6.0+PTX;6.1+PTX;7.0+PTX" + ;; + cpu) + unset FORCE_CUDA + export CUDA_VISIBLE_DEVICES= + ;; + *) + echo "Unrecognized CU_VERSION=$CU_VERSION" + exit 1 + ;; + esac +} + +setup_wheel_python() { + case "$PYTHON_VERSION" in + 3.6) python_abi=cp36-cp36m ;; + 3.7) python_abi=cp37-cp37m ;; + 3.8) python_abi=cp38-cp38 ;; + *) + echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION" + exit 1 + ;; + esac + export PATH="/opt/python/$python_abi/bin:$PATH" +} diff --git a/docs/notes/changelog.md b/docs/notes/changelog.md index 6381f3aa35..1bbd3d32e3 100644 --- a/docs/notes/changelog.md +++ b/docs/notes/changelog.md @@ -1,7 +1,10 @@ # Change Log +### Releases +See release log at +[https://github.com/facebookresearch/detectron2/releases](https://github.com/facebookresearch/detectron2/releases) -### Notable Changes: +### Notable Backward Incompatible Changes: * 2019-11-11: `detectron2.data.detection_utils.read_image` transposes images with exif information. * 2019-10-10: initial release. diff --git a/setup.py b/setup.py index e888651d3a..40db8844e4 100644 --- a/setup.py +++ b/setup.py @@ -20,9 +20,10 @@ def get_version(): version_line = [l.strip() for l in init_py if l.startswith("__version__")][0] version = version_line.split("=")[-1].strip().strip("'\"") - # Used by CI to build nightly packages. Users should never use it. - # To build a nightly wheel, run: - # FORCE_CUDA=1 BUILD_NIGHTLY=1 TORCH_CUDA_ARCH_LIST=All python setup.py bdist_wheel + # The following is used to build release packages. + # Users should never use it. + suffix = os.getenv("D2_VERSION_SUFFIX", "") + version = version + suffix if os.getenv("BUILD_NIGHTLY", "0") == "1": from datetime import datetime @@ -52,7 +53,9 @@ def get_extensions(): extra_compile_args = {"cxx": []} define_macros = [] - if (torch.cuda.is_available() and CUDA_HOME is not None) or os.getenv("FORCE_CUDA", "0") == "1": + if ( + torch.cuda.is_available() and CUDA_HOME is not None and os.path.isdir(CUDA_HOME) + ) or os.getenv("FORCE_CUDA", "0") == "1": extension = CUDAExtension sources += source_cuda define_macros += [("WITH_CUDA", None)] @@ -95,18 +98,20 @@ def get_model_zoo_configs() -> List[str]: path.dirname(path.realpath(__file__)), "detectron2", "model_zoo", "configs" ) # Symlink the config directory inside package to have a cleaner pip install. - if path.exists(destination): - # Remove stale symlink/directory from a previous build. + + # Remove stale symlink/directory from a previous build. + if path.exists(source_configs_dir): if path.islink(destination): os.unlink(destination) - else: + elif path.isdir(destination): shutil.rmtree(destination) - try: - os.symlink(source_configs_dir, destination) - except OSError: - # Fall back to copying if symlink fails: ex. on Windows. - shutil.copytree(source_configs_dir, destination) + if not path.exists(destination): + try: + os.symlink(source_configs_dir, destination) + except OSError: + # Fall back to copying if symlink fails: ex. on Windows. + shutil.copytree(source_configs_dir, destination) config_paths = glob.glob("configs/**/*.yaml", recursive=True) return config_paths @@ -124,7 +129,7 @@ def get_model_zoo_configs() -> List[str]: python_requires=">=3.6", install_requires=[ "termcolor>=1.1", - "Pillow==6.2.2", # torchvision currently does not work with Pillow 7 + "Pillow", # you can also use pillow-simd for better performance "yacs>=0.1.6", "tabulate", "cloudpickle",