From 2a4a97992c22abc9279615563cc6b78328ce6563 Mon Sep 17 00:00:00 2001 From: KleinYuan Date: Sun, 22 Jan 2023 13:52:30 -0800 Subject: [PATCH 1/3] fix docker build and update documentation --- INSTALL.md | 3 ++- docker/Dockerfile | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 8fb0c2171..27db84165 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -25,4 +25,5 @@ pip install -r docs/requirements.txt ``` # Set up with Dockder -comming soon + +Please check the [docker folder](docker) diff --git a/docker/Dockerfile b/docker/Dockerfile index 585412521..ff5682d17 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,8 @@ FROM nvidia/cuda:10.1-cudnn7-devel +# https://github.com/NVIDIA/nvidia-docker/issues/1632 +RUN rm /etc/apt/sources.list.d/cuda.list +RUN rm /etc/apt/sources.list.d/nvidia-ml.list ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && apt-get install -y \ python3-opencv ca-certificates python3-dev git wget sudo ninja-build @@ -12,8 +15,9 @@ RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER appuser WORKDIR /home/appuser +# https://github.com/facebookresearch/detectron2/issues/3933 ENV PATH="/home/appuser/.local/bin:${PATH}" -RUN wget https://bootstrap.pypa.io/get-pip.py && \ +RUN wget https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ python3 get-pip.py --user && \ rm get-pip.py From 4065f0a2429b0fceb31b78a426411a226a062e67 Mon Sep 17 00:00:00 2001 From: KleinYuan Date: Fri, 27 Jan 2023 14:41:37 -0800 Subject: [PATCH 2/3] add support for both cuda 10 and cuda 11 --- .dockerignore | 2 ++ .gitignore | 2 +- docker/Dockerfile => Dockerfile_CUDA10 | 13 +++++------- Dockerfile_CUDA11 | 28 +++++++++++++++++++++++++ docker/README.md | 22 ------------------- docker/README_DOCKER.md | 29 ++++++++++++++++++++++++++ docs/conf.py | 2 +- 7 files changed, 66 insertions(+), 32 deletions(-) create mode 100644 .dockerignore rename docker/Dockerfile => Dockerfile_CUDA10 (55%) create mode 100644 Dockerfile_CUDA11 delete mode 100644 docker/README.md create mode 100644 docker/README_DOCKER.md diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..95458c6ed --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +datasets/* +logs/* \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8be82c3e7..a2fc1b6a6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ - +datasets/* logs # compilation and distribution diff --git a/docker/Dockerfile b/Dockerfile_CUDA10 similarity index 55% rename from docker/Dockerfile rename to Dockerfile_CUDA10 index ff5682d17..c3a67946b 100644 --- a/docker/Dockerfile +++ b/Dockerfile_CUDA10 @@ -9,20 +9,17 @@ RUN apt-get update && apt-get install -y \ RUN ln -sv /usr/bin/python3 /usr/bin/python # create a non-root user -ARG USER_ID=1000 -RUN useradd -m --no-log-init --system --uid ${USER_ID} appuser -g sudo -RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers -USER appuser +COPY . /home/appuser WORKDIR /home/appuser # https://github.com/facebookresearch/detectron2/issues/3933 ENV PATH="/home/appuser/.local/bin:${PATH}" RUN wget https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ - python3 get-pip.py --user && \ + python3 get-pip.py && \ rm get-pip.py # install dependencies # See https://pytorch.org/ for other options if you use a different version of CUDA -RUN pip install --user tensorboard cmake # cmake from apt-get is too old -RUN pip install --user torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/cu101/torch_stable.html -RUN pip install --user -i https://pypi.tuna.tsinghua.edu.cn/simple tensorboard opencv-python cython yacs termcolor scikit-learn tabulate gdown gpustat faiss-gpu ipdb h5py +RUN pip install tensorboard cmake # cmake from apt-get is too old +RUN pip install torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/cu101/torch_stable.html +RUN pip install -i https://pypi.tuna.tsinghua.edu.cn/simple tensorboard opencv-python cython yacs termcolor scikit-learn tabulate gdown gpustat faiss-gpu ipdb h5py diff --git a/Dockerfile_CUDA11 b/Dockerfile_CUDA11 new file mode 100644 index 000000000..6f3424311 --- /dev/null +++ b/Dockerfile_CUDA11 @@ -0,0 +1,28 @@ +# FROM nvidia/cuda:10.1-cudnn7-devel +FROM nvidia/cuda:11.1.1-cudnn8-devel-ubuntu18.04 + +# https://github.com/NVIDIA/nvidia-docker/issues/1632 +#RUN rm /etc/apt/sources.list.d/cuda.list +#RUN rm /etc/apt/sources.list.d/nvidia-ml.list +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y \ + python3-opencv ca-certificates python3-dev git wget sudo ninja-build +RUN ln -sv /usr/bin/python3 /usr/bin/python + +# create a non-root user +COPY . /home/appuser +WORKDIR /home/appuser + +# https://github.com/facebookresearch/detectron2/issues/3933 +ENV PATH="/home/appuser/.local/bin:${PATH}" +RUN wget https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ + python3 get-pip.py && \ + rm get-pip.py + +# install dependencies +# See https://pytorch.org/ for other options if you use a different version of CUDA +RUN pip install tensorboard cmake # cmake from apt-get is too old +RUN pip install torch==1.10 torchvision==0.11.1 -f https://download.pytorch.org/whl/cu111/torch_stable.html +# RUN pip install torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/cu101/torch_stable.html +# RUN pip install -i https://pypi.tuna.tsinghua.edu.cn/simple tensorboard opencv-python cython yacs termcolor scikit-learn tabulate gdown gpustat faiss-gpu ipdb h5py +RUN pip install tensorboard opencv-python cython yacs termcolor scikit-learn tabulate gdown gpustat faiss-gpu ipdb h5py diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index 351b30d09..000000000 --- a/docker/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Use the container - -```shell script -cd docker/ -# Build: -docker build -t=fastreid:v0 . -# Launch (requires GPUs) -nvidia-docker run -v server_path:docker_path --name=fastreid --net=host --ipc=host -it fastreid:v0 /bin/sh -``` - -## Install new dependencies - -Add the following to `Dockerfile` to make persist changes. -```shell script -RUN sudo apt-get update && sudo apt-get install -y vim -``` - -Or run them in the container to make temporary changes. - -## A more complete docker container - -If you want to use a complete docker container which contains many useful tools, you can check my development environment [Dockerfile](https://github.com/L1aoXingyu/fastreid_docker) \ No newline at end of file diff --git a/docker/README_DOCKER.md b/docker/README_DOCKER.md new file mode 100644 index 000000000..a120ab995 --- /dev/null +++ b/docker/README_DOCKER.md @@ -0,0 +1,29 @@ +# Use the container + +### Build Container + +```shell script +cd docker/ +# Build with the corresponding CUDA version + +# CUDA 10 +docker build -t=fastreid:v0 -f Dockerfile_CUDA10 . +# CUDA 11 +docker build -t=fastreid:v0 -f Dockerfile_CUDA11 . +``` + +### Run Container + +``` +# Launch (requires GPUs) +nvidia-docker run -v ${PWD}:/home/appuser --name=fastreid --net=host --ipc=host -it fastreid:v0 +``` + +### Run Training + +Next, follow the [Get Started Doc](https://github.com/JDAI-CV/fast-reid/blob/master/GETTING_STARTED.md#compile-with-cython-to-accelerate-evalution). + + +## A more complete docker container + +If you want to use a complete docker container which contains many useful tools, you can check my development environment [Dockerfile](https://github.com/L1aoXingyu/fastreid_docker) diff --git a/docs/conf.py b/docs/conf.py index aad5c16d9..c2007641f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -161,7 +161,7 @@ def resolve_any_xref(self, env, fromdocname, builder, target, node, contnode): # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "build", "README.md", "tutorials/README.md"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "build", "README_DOCKER.md", "tutorials/README_DOCKER.md"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" From df3d23bbd275471a701ac7f9c6c702cdb113ed47 Mon Sep 17 00:00:00 2001 From: KleinYuan Date: Fri, 27 Jan 2023 14:58:35 -0800 Subject: [PATCH 3/3] clean up --- .dockerignore | 2 +- docker/{README_DOCKER.md => README.md} | 0 docs/conf.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename docker/{README_DOCKER.md => README.md} (100%) diff --git a/.dockerignore b/.dockerignore index 95458c6ed..5d36f76ab 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,2 @@ datasets/* -logs/* \ No newline at end of file +logs/* diff --git a/docker/README_DOCKER.md b/docker/README.md similarity index 100% rename from docker/README_DOCKER.md rename to docker/README.md diff --git a/docs/conf.py b/docs/conf.py index c2007641f..aad5c16d9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -161,7 +161,7 @@ def resolve_any_xref(self, env, fromdocname, builder, target, node, contnode): # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "build", "README_DOCKER.md", "tutorials/README_DOCKER.md"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "build", "README.md", "tutorials/README.md"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx"