Skip to content

Commit

Permalink
Version 0.8
Browse files Browse the repository at this point in the history
  • Loading branch information
@yairomer
yairomer committed Aug 21, 2021
1 parent 56a21b5 commit 257249c
Show file tree
Hide file tree
Showing 3 changed files with 112 additions and 89 deletions.
141 changes: 60 additions & 81 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04
FROM nvidia/cuda:11.0-cudnn8-devel-ubuntu18.04

## Install basic packages and useful utilities
## ===========================================
Expand All @@ -18,12 +18,9 @@ RUN apt-get update -y && \
cmake \
sudo \
openssh-server \
python3 \
python3-dev \
python3.8 \
python3.8-dev \
python3-pip \
python \
python-dev \
python-pip \
pylint \
sshfs \
wget \
Expand Down Expand Up @@ -51,7 +48,6 @@ RUN apt-get update -y && \
libfreetype6-dev \
libpng-dev \
ffmpeg \
python-qt4 \
python3-pyqt5 \
imagemagick \
inkscape \
Expand All @@ -64,7 +60,6 @@ RUN apt-get update -y && \
pandoc \
texlive-fonts-extra \
texlive-fonts-recommended \
texlive-generic-recommended \
texlive-latex-base \
texlive-latex-extra \
texlive-xetex \
Expand All @@ -73,7 +68,7 @@ RUN apt-get update -y && \
libncursesw5-dev \
&& \
apt-get install -y neovim && \
pip install pynvim==0.3.2 && \
pip3 install pynvim==0.3.2 && \
apt-get clean

## ToDo: increase memory limit to 10GB in: /etc/ImageMagick-6/policy.xml
Expand All @@ -98,8 +93,8 @@ RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
## ==========
RUN mkdir /var/run/sshd && \
sed 's/^#\?PasswordAuthentication .*$/PasswordAuthentication yes/g' -i /etc/ssh/sshd_config && \
sed 's/^#\?Port .*$/Port 9022/g' -i /etc/ssh/sshd_config && \
sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd && \
echo "user_allow_other" >> /etc/fuse.conf

## VSCode
## ======
Expand All @@ -112,107 +107,91 @@ RUN cd /tmp && \
apt-get install -y code && \
rm microsoft.gpg

## Install pycharm
## ===============
ARG PYCHARM_SOURCE="https://download.jetbrains.com/python/pycharm-community-2018.3.3.tar.gz"
RUN mkdir /opt/pycharm && \
cd /opt/pycharm && \
curl -L $PYCHARM_SOURCE -o installer.tgz && \
tar --strip-components=1 -xzf installer.tgz && \
rm installer.tgz && \
/usr/bin/python2 /opt/pycharm/helpers/pydev/setup_cython.py build_ext --inplace && \
/usr/bin/python3 /opt/pycharm/helpers/pydev/setup_cython.py build_ext --inplace
COPY ./resources/pycharm.bin /usr/local/bin/pycharm

## Setup app folder
## ================
RUN mkdir /app && \
chmod 777 /app

## Setup python environment
## ========================
RUN pip3 install pip==20.0.2 && \
hash -r pip && \
pip3 install -U \
virtualenv==20.0.10 \
ipython==7.13.0 \
numpy==1.18.1 \
scipy==1.4.0 \
cvxpy==1.0.28 \
matplotlib==3.2.0 \
PyQt5==5.14.1 \
seaborn==0.10.0 \
plotly==4.5.3 \
dash==1.9.1 \
bokeh==2.0.0 \
RUN python3.8 -m pip install pip==21.0.1 && \
python3.8 -m pip install wrapt --ignore-installed && \
python3.8 -m pip install -U \
altair==4.1.0 \
bokeh==2.3.1 \
chainer==7.7.0 \
cvxpy==1.1.12 \
dash==1.20.0 \
filelock==3.0.12 \
flake8==3.9.1 \
Flask==1.1.2 \
ggplot==0.11.5 \
altair==4.0.1 \
pandas==1.0.1 \
pyyaml==5.3 \
protobuf==3.11.3 \
ipdb==0.13.2 \
flake8==3.7.9 \
cython==0.29.15 \
sympy==1.5.1 \
nose==1.3.7 \
sphinx==1.8.1 \
tqdm==4.43.0 \
opencv-contrib-python==4.2.0.32 \
scikit-image==0.16.2 \
scikit-learn==0.22.2 \
imageio==2.8.0 \
torchvision==0.4.0 \
torchviz==0.0.1 \
Pillow==6.1 \
torchsummary==1.5.1 \
tensorflow-gpu==2.0.0 \
tensorboardX==2.0 \
graphviz==0.16 \
h5py==2.10.0 \
imageio==2.9.0 \
ipdb==0.13.7 \
ipython==7.22.0 \
ipywidgets==7.6.3 \
jupyter==1.0.0 \
jupyter-contrib-nbextensions==0.5.1 \
jupyterlab==3.0.14 \
jupyterthemes==0.20.0 \
jupyter_contrib_nbextensions==0.5.1 \
jupyterlab==2.0.1 \
ipywidgets==7.5.1 \
kaleido==0.2.1 \
line-profiler==3.2.1 \
lxml==4.6.3 \
matplotlib==3.4.1 \
nbconvert==6.0.7 \
nose==1.3.7 \
numpy==1.19.5 \
opencv-contrib-python==4.5.1.48 \
pandas==1.2.4 \
Pillow==8.2.0 \
plotly==4.14.3 \
protobuf==3.15.8 \
pylint==2.8.2 \
PyQt5==5.15.4 \
PyYAML==5.4.1 \
scikit-image==0.18.1 \
scikit-learn==0.24.2 \
scipy==1.6.3 \
seaborn==0.11.1 \
Sphinx==3.5.4 \
sympy==1.8 \
tensorboardX==2.2 \
tensorflow-gpu==2.4.1 \
torchaudio==0.8.1 \
torchsummary==1.5.1 \
torchvision==0.9.1 \
torchviz==0.0.2 \
tqdm==4.60.0 \
virtualenv==20.4.4 \
visdom==0.1.8.9 \
line_profiler==3.0.2 \
&& \
rm -r /root/.cache/pip
ENV MPLBACKEND=Agg

## Import matplotlib the first time to build the font cache.
## ---------------------------------------------------------
RUN python3 -c "import matplotlib.pyplot" && \
RUN python3.8 -c "import matplotlib.pyplot" && \
cp -r /root/.cache /etc/skel/

## Setup Jupyter
## -------------
RUN pip install six==1.11 && \
RUN python3.8 -m pip install six==1.11 && \
jupyter nbextension enable --py widgetsnbextension && \
jupyter contrib nbextension install --system && \
jupyter nbextensions_configurator enable && \
jupyter serverextension enable --py jupyterlab --system && \
pip install RISE && \
python3.8 -m pip install RISE && \
jupyter-nbextension install rise --py --sys-prefix --system && \
cp -r /root/.jupyter /etc/skel/

## Install Orca (for exporting Plotly figures to images)
## =====================================================
RUN apt install -y xvfb libgconf2-4 && \
wget https://github.com/plotly/orca/releases/download/v1.1.1/orca-1.1.1-x86_64.AppImage -P /tmp && \
chmod 777 /tmp/orca-1.1.1-x86_64.AppImage && \
cd /tmp && \
./orca-1.1.1-x86_64.AppImage --appimage-extract && \
mv /tmp/squashfs-root /opt/squashfs-root && \
chmod -R 777 /opt/squashfs-root && \
printf '#!/bin/bash \nxvfb-run --auto-servernum --server-args "-screen 0 640x480x24" /opt/squashfs-root/app/orca "$@"' > /usr/bin/orca && \
chmod 777 /usr/bin/orca && \
rm -r /tmp/orca-1.1.1-x86_64.AppImage

## Create virtual environment
## ==========================
RUN cd /app/ && \
virtualenv --system-site-packages dockvenv && \
grep -rlnw --null /usr/local/bin/ -e '#!/usr/bin/python3' | xargs -0r cp -t /app/dockvenv/bin/ && \
sed -i "s/#"'!'"\/usr\/bin\/python3/#"'!'"\/usr\/bin\/env python/g" /app/dockvenv/bin/* && \
virtualenv --python=python3.8 --system-site-packages /app/dockvenv && \
grep -rlnw --null /usr/local/bin/ -e '#!/usr/bin/python3.8' | xargs -0r cp -t /app/dockvenv/bin/ && \
sed -i "s/#"'!'"\/usr\/bin\/python3.8/#"'!'"\/usr\/bin\/env python/g" /app/dockvenv/bin/* && \
mv /app/dockvenv /root/ && \
ln -sfT /root/dockvenv /app/dockvenv && \
cp -rp /root/dockvenv /etc/skel/ && \
Expand Down
53 changes: 45 additions & 8 deletions mldock.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@ app_name=mldock

repository="omeryair/"
image_name="mldock"
version_name="v0.5"
version_name="v0.8"

container_name="mldock_$USER"
container_name=""
pass_ssh_key=false

main_cli() {
## Parse args
Expand All @@ -23,6 +24,7 @@ main_cli() {
echo " setup Create a link or a copy of the $app_name.sh script in the /usr/bin folder (requiers sudo)."
echo " build Build the image."
echo " run Run a command inside a new container."
echo " run_server Run the default mldock server."
echo " run_remote Run a command inside a new container on a remote machine."
echo " exec Execute a command inside an existing container."
echo " exec_remote Execute a command inside an existing container on a remote machine."
Expand Down Expand Up @@ -69,6 +71,9 @@ main_cli() {
run)
run_cli "$@"
;;
run_server)
run_server_cli "$@"
;;
run_remote)
run_remote_cli "$@"
;;
Expand Down Expand Up @@ -196,7 +201,7 @@ run_cli() {
fi
map_host=true
detach_container=false
home_folder=""
home_folder="$HOME"
working_folder=""
command_to_run=""
use_tmux=false
Expand All @@ -219,7 +224,7 @@ run_cli() {
echo "Options:"
echo " -v version_name The version name to use for the build image. Default: \"$version_name\""
echo " -c container_name The name to for the created container. Default: \"$container_name\""
echo " -f home_folder A folder to map as the dockuser's home folder."
echo " -f home_folder A folder to map as the dockuser's home folder. Default: \"$home_folder\""
echo " -w working_folder The working folder."
echo " -s Run the command as root."
echo " -u Run the command as dockuser user."
Expand Down Expand Up @@ -301,6 +306,25 @@ run_cli() {
run_command
}

run_server_cli() {
container_name="mldock_$USER"

usage () {
echo "Run the default mldock server"
echo ""
echo "This command is equivalent to runing \"mldock run -d -c $container_name run_server\"."
echo "This command share the same arguments of the \"run\" command."
}

if [ "$#" -eq 1 ] && [ "$1" == "-h" ]; then
usage
exit 0
fi

set -- "$@" -d -c "$container_name" run_server
run_cli
}

run_remote_cli() {
if xhost >& /dev/null ; then
## Display exist
Expand All @@ -316,7 +340,8 @@ run_remote_cli() {
fi
map_host=true
detach_container=false
home_folder=""
local_ip=`hostname -I | cut -d " " -f1`
home_folder="$USER@${local_ip}:$HOME"
working_folder=""
command_to_run=""
use_tmux=false
Expand All @@ -329,7 +354,7 @@ run_remote_cli() {
echo "Options:"
echo " -v version_name The version name to use for the build image. Default: \"$version_name\""
echo " -c container_name The name to for the created container. Default: \"$container_name\""
echo " -f home_folder A folder to map as the dockuser's home folder."
echo " -f home_folder A folder to map as the dockuser's home folder. Default: \"$home_folder\""
echo " -w working_folder The working folder."
echo " -s Run the command as root."
echo " -u Run the command as dockuser user."
Expand Down Expand Up @@ -424,6 +449,10 @@ run_remote_cli() {
else
use_nvidia_runtime=false
fi
if [ -f "$HOME/.ssh/id_rsa.mldock" ]; then
pass_ssh_key=true
fi

check_docker_for_sudo
gen_command
run_remote_command
Expand Down Expand Up @@ -663,12 +692,20 @@ gen_command() {
extra_args+=("--runtime=nvidia")
fi
fi
if [ "$pass_ssh_key" = true ]; then
ssh_key=`cat $HOME/.ssh/id_rsa.mldock`
ssh_key="${ssh_key//$'\n'/ }"
extra_args+=("-e" "SSHKEY=\"$ssh_key\"")
fi
cmd+=("docker" "run" \
"-e" "SSHPORT=9022" \
"--rm" \
"--privileged" \
"--network" "host" \
"--name" "$container_name")
"--shm-size=2gb" \
"--network=host")
if [[ ! -z "$container_name" ]]; then
cmd+=( "--name=$container_name" )
fi
cmd+=( "${extra_args[@]}" )
cmd+=("$repository$image_name:$version_name")
if [ "$use_tmux" = true ]; then
Expand Down
7 changes: 7 additions & 0 deletions resources/switch_user_run.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,13 @@ if [ ! -z "$SSHPORT" ]; then
/etc/init.d/ssh restart > /dev/null 2>&1
fi

if [ ! -z "$SSHKEY" ]; then
mkdir /root/.ssh
echo "${SSHKEY// /$'\n'}" > /root/.ssh/id_rsa.mldock
chmod 400 /root/.ssh/id_rsa.mldock
echo "IdentityFile ~/yy.ssh/id_rsa.mldock" >> /root/.ssh/config
fi

if [ ! -z "$SSHFSDIRS" ]; then
IFS=',' read -ra SSHFSDIRS <<< "$SSHFSDIRS"
for map in "${SSHFSDIRS[@]}"; do
Expand Down

0 comments on commit 257249c

Please sign in to comment.