@allow-large-files [vissl][PR] V0.1.6 build fairscale pip pckg @allow-large-files (#465)

Summary:
1. Add fairscale to pip binary, as pypi won't allow you to have direct dependencies.
2. Change tutorials / installation from source instructions to reflect above.
3. I also moved up conda installation instructions to be first, since I think we should promote using conda (easier to manage environments).

Pull Request resolved: #465

Reviewed By: QuentinDuval

Differential Revision: D32069671

Pulled By: iseessel

fbshipit-source-id: 54180d5882afa351c4c725bf4dbfa228b38cd7d5

Author: iseessel

.circleci/config.yml

@@ -54,6 +54,15 @@ update_gcc7: &update_gcc7
         gcc --version
         g++ --version
 
+install_fairscale: &install_fairscale
+  - run:
+      name: Install Fairscale
+      working_directory: ~/
+      command: |
+        pip uninstall -y fairscale
+        pip install fairscale@https://github.com/facebookresearch/fairscale/tarball/df7db85cef7f9c30a5b821007754b96eb1f977b6
+
+
 install_classy_vision: &install_classy_vision
   - run:
       name: Install ClassyVision
@@ -170,6 +179,7 @@ jobs:
       - <<: *install_vissl_dep
       - <<: *install_augly
       - <<: *install_classy_vision
+      - <<: *install_fairscale
       - <<: *install_apex_cpu
       - <<: *pip_list
 
@@ -213,6 +223,7 @@ jobs:
 
       - <<: *install_vissl_dep
       - <<: *install_classy_vision
+      - <<: *install_fairscale
       - <<: *update_gcc7
       - <<: *install_apex_gpu
       - <<: *pip_list

INSTALL.md

@@ -26,24 +26,27 @@ At a high level, project requires following system dependencies.
 ## Installing VISSL from source (recommended)
 The following instructions assume that you have desired CUDA version installed and working.
 
-### Install from source in PIP environment
+### Install from source in Conda environment
+
+#### Step 1: Create Conda environment
+
+If you don't have anaconda, [run this bash scrip to install conda](https://github.com/facebookresearch/vissl/blob/main/docker/common/install_conda.sh).
 
-#### Step 1: Create Virtual environment (pip)
 ```bash
-python3 -m venv ~/venv
-. ~/venv/bin/activate
+conda create -n vissl_env python=3.7
+source activate vissl_env
 ```
 
-#### Step 2: Install PyTorch (pip)
+#### Step 2: Install PyTorch (conda)
 
 ```bash
-pip install torch==1.7.1+cu101 torchvision==0.8.2+cu101 -f https://download.pytorch.org/whl/torch_stable.html
+conda install pytorch torchvision cudatoolkit=10.1 -c pytorch
 ```
 
-#### Step 3: Install APEX (pip)
+#### Step 3: Install APEX (conda)
 
 ```bash
-pip install -f https://dl.fbaipublicfiles.com/vissl/packaging/apexwheels/py37_cu101_pyt171/download.html apex
+conda install -c vissl apex
 ```
 
 #### Step 4: Install VISSL
@@ -58,36 +61,37 @@ git checkout -b v0.1.6
 # install vissl dependencies
 pip install --progress-bar off -r requirements.txt
 pip install opencv-python
-# update classy vision install to current main branch.
+# update classy vision install to commit stable for vissl.
+# Note: If building from vissl main, use classyvision main.
 pip uninstall -y classy_vision
-pip install classy-vision@https://github.com/facebookresearch/ClassyVision/tarball/main
+pip install classy-vision@https://github.com/facebookresearch/ClassyVision/tarball/4785d5ee19d3bcedd5b28c1eb51ea1f59188b54d
+# update fairscale install to commit stable for vissl.
+pip uninstall -y fairscale
+pip install fairscale@https://github.com/facebookresearch/fairscale/tarball/df7db85cef7f9c30a5b821007754b96eb1f977b6
 # install vissl dev mode (e stands for editable)
 pip install -e ".[dev]"
 # verify installation
 python -c 'import vissl, apex'
 ```
 
-### Install from source in Conda environment
-
-#### Step 1: Create Conda environment
-
-If you don't have anaconda, [run this bash scrip to install conda](https://github.com/facebookresearch/vissl/blob/main/docker/common/install_conda.sh).
+### Install from source in PIP environment
 
+#### Step 1: Create Virtual environment (pip)
 ```bash
-conda create -n vissl_env python=3.7
-source activate vissl_env
+python3 -m venv ~/venv
+. ~/venv/bin/activate
 ```
 
-#### Step 2: Install PyTorch (conda)
+#### Step 2: Install PyTorch (pip)
 
 ```bash
-conda install pytorch torchvision cudatoolkit=10.1 -c pytorch
+pip install torch==1.7.1+cu101 torchvision==0.8.2+cu101 -f https://download.pytorch.org/whl/torch_stable.html
 ```
 
-#### Step 3: Install APEX (conda)
+#### Step 3: Install APEX (pip)
 
 ```bash
-conda install -c vissl apex
+pip install -f https://dl.fbaipublicfiles.com/vissl/packaging/apexwheels/py37_cu101_pyt171/download.html apex
 ```
 
 #### Step 4: Install VISSL

dev/packaging/vissl_pip/go.sh

@@ -9,9 +9,15 @@ git clone https://github.com/facebookresearch/ClassyVision.git
 cd ClassyVision || exit
 git reset --hard 4785d5ee19d3bcedd5b28c1eb51ea1f59188b54d
 cd ../ || exit
-
 rm -rf ../../../classy_vision
 cp -r ClassyVision/classy_vision ../../../classy_vision
+
+rm -rf fairscale
+git clone https://github.com/facebookresearch/fairscale.git
+cd fairscale || exit
+git reset --hard df7db85cef7f9c30a5b821007754b96eb1f977b6
+cd ../ || exit
 rm -rf ../../../fairscale
+cp -r fairscale/fairscale ../../../fairscale
 
-docker run --rm  -v "$PWD/../../..:/inside pytorch/conda-cuda bash inside/dev/packaging/vissl_pip/inside.sh"
+docker run --rm  -v $PWD/../../..:/inside pytorch/conda-cuda bash inside/dev/packaging/vissl_pip/inside.sh

requirements.txt

@@ -1,7 +1,6 @@
 # Keep this sorted for easy search.
 # Keep versions pinned for testing reproducibility.
 cython==0.29.22
-fairscale@https://github.com/facebookresearch/fairscale/tarball/df7db85cef7f9c30a5b821007754b96eb1f977b6
 fvcore==0.1.3.post20210317
 iopath==0.1.9
 hydra-core==1.0.7

tutorials/Benchmark_Full_Finetuning_on_ImageNet_1K_V0_1_6.ipynb

@@ -87,6 +87,10 @@
         "!pip uninstall -y classy_vision\n",
         "!pip install classy-vision@https://github.com/facebookresearch/ClassyVision/tarball/4785d5ee19d3bcedd5b28c1eb51ea1f59188b54d\n",
         "\n",
+        "# Update fairscale to commit compatible with v0.1.6\n",
+        "!pip uninstall -y fairscale\n",
+        "!pip install fairscale@https://github.com/facebookresearch/fairscale/tarball/df7db85cef7f9c30a5b821007754b96eb1f977b6\n",
+        "\n",
         "# install vissl dev mode (e stands for editable)\n",
         "!pip install -e .[dev]"
       ]

tutorials/Benchmark_Linear_Image_Classification_on_ImageNet_1K_V0_1_6.ipynb

@@ -87,6 +87,10 @@
         "!pip uninstall -y classy_vision\n",
         "!pip install classy-vision@https://github.com/facebookresearch/ClassyVision/tarball/4785d5ee19d3bcedd5b28c1eb51ea1f59188b54d\n",
         "\n",
+        "# Update fairscale to commit compatible with v0.1.6\n",
+        "!pip uninstall -y fairscale\n",
+        "!pip install fairscale@https://github.com/facebookresearch/fairscale/tarball/df7db85cef7f9c30a5b821007754b96eb1f977b6\n",
+        "\n",
         "# install vissl dev mode (e stands for editable)\n",
         "!pip install -e .[dev]"
       ]

tutorials/Feature_Extraction_V0_1_6.ipynb

@@ -94,6 +94,10 @@
         "!pip uninstall -y classy_vision\n",
         "!pip install classy-vision@https://github.com/facebookresearch/ClassyVision/tarball/4785d5ee19d3bcedd5b28c1eb51ea1f59188b54d\n",
         "\n",
+        "# Update fairscale to commit compatible with v0.1.6\n",
+        "!pip uninstall -y fairscale\n",
+        "!pip install fairscale@https://github.com/facebookresearch/fairscale/tarball/df7db85cef7f9c30a5b821007754b96eb1f977b6\n",
+        "\n",
         "# install vissl dev mode (e stands for editable)\n",
         "!pip install -e .[dev]"
       ]

tutorials/Installation_V0_1_6.ipynb

@@ -181,6 +181,9 @@
         "# update classy vision install to current main.\n",
         "pip uninstall -y classy_vision\n",
         "pip install classy-vision@https://github.com/facebookresearch/ClassyVision/tarball/main\n",
+        "# Update fairscale to commit compatible with vissl main.\n",
+        "!pip uninstall -y fairscale\n",
+        "!pip install fairscale@https://github.com/facebookresearch/fairscale/tarball/df7db85cef7f9c30a5b821007754b96eb1f977b6\n",
         "# install vissl dev mode (e stands for editable)\n",
         "pip install -e .[dev]\n",
         "# verify installation\n",

tutorials/Train_SimCLR_on_1_gpu_V0_1_6.ipynb

@@ -94,6 +94,10 @@
         "!pip uninstall -y classy_vision\n",
         "!pip install classy-vision@https://github.com/facebookresearch/ClassyVision/tarball/4785d5ee19d3bcedd5b28c1eb51ea1f59188b54d\n",
         "\n",
+        "# Update fairscale to commit compatible with v0.1.6\n",
+        "!pip uninstall -y fairscale\n",
+        "!pip install fairscale@https://github.com/facebookresearch/fairscale/tarball/df7db85cef7f9c30a5b821007754b96eb1f977b6\n",
+        "\n",
         "# install vissl dev mode (e stands for editable)\n",
         "!pip install -e .[dev]"
       ]

tutorials/Understanding_VISSL_Training_and_YAML_Config_V0_1_6.ipynb

@@ -91,6 +91,10 @@
         "!pip uninstall -y classy_vision\n",
         "!pip install classy-vision@https://github.com/facebookresearch/ClassyVision/tarball/4785d5ee19d3bcedd5b28c1eb51ea1f59188b54d\n",
         "\n",
+        "# Update fairscale to commit compatible with v0.1.6\n",
+        "!pip uninstall -y fairscale\n",
+        "!pip install fairscale@https://github.com/facebookresearch/fairscale/tarball/df7db85cef7f9c30a5b821007754b96eb1f977b6\n",
+        "\n",
         "# install vissl dev mode (e stands for editable)\n",
         "!pip install -e .[dev]"
       ]