Skip to content

Commit

Permalink
Merge pull request #5 from UCSB-PSTAT/develop
Browse files Browse the repository at this point in the history 
Lets Build Again
  • Loading branch information
@vwbusguy
vwbusguy authored Oct 16, 2024
2 parents 4dd58f5 + 9b2ebe3 commit a495930
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 26 deletions.
16 changes: 9 additions & 7 deletions Containerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,21 +23,23 @@ RUN wget https://sjtodd.github.io/ling110/srilm-1.7.3.tar.gz && \
make MAKE_PIC=yes World && \
make cleanest

RUN pip install gensim \
RUN mamba install -y \
gensim \
scikit-learn \
pytest \
PTable \
ptable \
nltk \
arpa \
morfessor \
tensorflow \
pytorch \
tensorflow-cpu \
torchaudio \
torchvision \
keras \
spacy \
textgrid \
tgt
textgrid

# Because Pytorch is special:
RUN pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
RUN pip install arpa tgt

# Hopefully temporary workaround
RUN R -e "install.packages('rtkore',repos = 'https://cloud.r-project.org/', Ncpus = parallel::detectCores()); install.packages('https://cran.r-project.org/src/contrib/Archive/blockcluster/blockcluster_4.5.4.tar.gz', repos=NULL, type='source', Ncpus = parallel::detectCores())"
Expand Down
47 changes: 28 additions & 19 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,10 @@ pipeline {
stages {
stage('Build Test Deploy') {
agent {
label 'jupyter'
kubernetes {
cloud 'rke-test'
inheritFrom 'podman'
}
}
stages{
stage('Build') {
Expand All @@ -19,34 +22,38 @@ pipeline {
scmSkip(deleteBuild: true, skipPattern:'.*\\[ci skip\\].*')
}
}
echo "NODE_NAME = ${env.NODE_NAME}"
sh 'podman build -t localhost/$IMAGE_NAME --pull --force-rm --no-cache .'
container('podman') {
echo "NODE_NAME = ${env.NODE_NAME}"
sh 'podman build -t localhost/$IMAGE_NAME --pull --force-rm --no-cache .'
}
}
post {
unsuccessful {
sh 'podman rmi -i localhost/$IMAGE_NAME || true'
container('podman') {sh 'podman rmi -i localhost/$IMAGE_NAME || true'}
}
}
}
stage('Test') {
steps {
sh 'podman run -it --rm --pull=never localhost/$IMAGE_NAME which rstudio'
sh 'podman run -it --rm --pull=never localhost/$IMAGE_NAME which ngram'
sh 'podman run -it --rm --pull=never localhost/$IMAGE_NAME R -q -e "getRversion() >= \\"4.1.3\\"" | tee /dev/stderr | grep -q "TRUE"'
sh 'podman run -it --rm --pull=never localhost/$IMAGE_NAME python -c "import numpy;import gensim;import sklearn; sklearn.show_versions();import pytest;from prettytable import PrettyTable;import requests;import matplotlib;import nltk;import pandas;import arpa;import morfessor;import spacy; import torch; import keras; import tensorflow; import tgt; import textgrid;"'
container('podman') {
sh 'podman run -it --rm --pull=never localhost/$IMAGE_NAME which rstudio'
sh 'podman run -it --rm --pull=never localhost/$IMAGE_NAME which ngram'
sh 'podman run -it --rm --pull=never localhost/$IMAGE_NAME R -q -e "getRversion() >= \\"4.1.3\\"" | tee /dev/stderr | grep -q "TRUE"'
sh 'podman run -it --rm --pull=never localhost/$IMAGE_NAME python -c "import numpy;import gensim;import sklearn; sklearn.show_versions();import pytest;from prettytable import PrettyTable;import requests;import matplotlib;import nltk;import pandas;import arpa;import morfessor;import spacy; import torch; import keras; import tensorflow; import tgt; import textgrid;"'

sh 'podman run -it --rm --pull=never localhost/$IMAGE_NAME R -e "library(\"bayestestR\");library(\"blockcluster\");library(\"ca\");library(\"CCA\");library(\"cowplot\");library(\"DirichletReg\");library(\"doParallel\");library(\"ellipse\");library(\"factoextra\");library(\"FactoMineR\");library(\"ggalluvial\");library(\"GGally\");library(\"ggbreak\");library(\"ggfittext\");library(\"ggforce\");library(\"ggmosaic\");library(\"ggplot2\");library(\"ggrepel\");library(\"glossr\");library(\"keras\");library(\"lmerTest\");library(\"mclust\");library(\"ordinal\");library(\"plotly\");library(\"MASS\");library(\"pvclust\");library(\"reticulate\");library(\"rjson\");library(\"see\");library(\"spacyr\");library(\"vowels\");library(\"rezonateR\")"'
sh 'podman run -d --name=$IMAGE_NAME --rm --pull=never -p 8888:8888 localhost/$IMAGE_NAME start-notebook.sh --NotebookApp.token="jenkinstest"'
sh 'sleep 10 && curl -v http://localhost:8888/rstudio?token=jenkinstest 2>&1 | grep -P "HTTP\\S+\\s[1-3][0-9][0-9]\\s+[\\w\\s]+\\s*$"'
sh 'curl -v http://localhost:8888/lab?token=jenkinstest 2>&1 | grep -P "HTTP\\S+\\s200\\s+[\\w\\s]+\\s*$"'
sh 'curl -v http://localhost:8888/tree?token=jenkinstest 2>&1 | grep -P "HTTP\\S+\\s200\\s+[\\w\\s]+\\s*$"'
sh 'podman run -it --rm --pull=never localhost/$IMAGE_NAME R -e "library(\"bayestestR\");library(\"blockcluster\");library(\"ca\");library(\"CCA\");library(\"cowplot\");library(\"DirichletReg\");library(\"doParallel\");library(\"ellipse\");library(\"factoextra\");library(\"FactoMineR\");library(\"ggalluvial\");library(\"GGally\");library(\"ggbreak\");library(\"ggfittext\");library(\"ggforce\");library(\"ggmosaic\");library(\"ggplot2\");library(\"ggrepel\");library(\"glossr\");library(\"keras\");library(\"lmerTest\");library(\"mclust\");library(\"ordinal\");library(\"plotly\");library(\"MASS\");library(\"pvclust\");library(\"reticulate\");library(\"rjson\");library(\"see\");library(\"spacyr\");library(\"vowels\");library(\"rezonateR\")"'
sh 'podman run -d --name=$IMAGE_NAME --rm --pull=never -p 8888:8888 localhost/$IMAGE_NAME start-notebook.sh --NotebookApp.token="jenkinstest"'
sh 'sleep 10 && curl -v http://localhost:8888/rstudio?token=jenkinstest 2>&1 | grep -P "HTTP\\S+\\s[1-3][0-9][0-9]\\s+[\\w\\s]+\\s*$"'
sh 'curl -v http://localhost:8888/lab?token=jenkinstest 2>&1 | grep -P "HTTP\\S+\\s200\\s+[\\w\\s]+\\s*$"'
sh 'curl -v http://localhost:8888/tree?token=jenkinstest 2>&1 | grep -P "HTTP\\S+\\s200\\s+[\\w\\s]+\\s*$"'
}
}
post {
always {
sh 'podman rm -ifv $IMAGE_NAME'
container('podman') {sh 'podman rm -ifv $IMAGE_NAME'}
}
unsuccessful {
sh 'podman rmi -i localhost/$IMAGE_NAME || true'
container('podman') {sh 'podman rmi -i localhost/$IMAGE_NAME || true'}
}
}
}
Expand All @@ -56,19 +63,21 @@ pipeline {
DOCKER_HUB_CREDS = credentials('DockerHubToken')
}
steps {
sh 'skopeo copy containers-storage:localhost/$IMAGE_NAME docker://docker.io/ucsb/$IMAGE_NAME:latest --dest-username $DOCKER_HUB_CREDS_USR --dest-password $DOCKER_HUB_CREDS_PSW'
sh 'skopeo copy containers-storage:localhost/$IMAGE_NAME docker://docker.io/ucsb/$IMAGE_NAME:v$(date "+%Y%m%d") --dest-username $DOCKER_HUB_CREDS_USR --dest-password $DOCKER_HUB_CREDS_PSW'
container('podman') {
sh 'skopeo copy containers-storage:localhost/$IMAGE_NAME docker://docker.io/ucsb/$IMAGE_NAME:latest --dest-username $DOCKER_HUB_CREDS_USR --dest-password $DOCKER_HUB_CREDS_PSW'
sh 'skopeo copy containers-storage:localhost/$IMAGE_NAME docker://docker.io/ucsb/$IMAGE_NAME:v$(date "+%Y%m%d") --dest-username $DOCKER_HUB_CREDS_USR --dest-password $DOCKER_HUB_CREDS_PSW'
}
}
post {
always {
sh 'podman rmi -i localhost/$IMAGE_NAME || true'
container('podman') {sh 'podman rmi -i localhost/$IMAGE_NAME || true'}
}
}
}
}
post {
always {
sh 'podman rmi -i localhost/$IMAGE_NAME || true'
container('podman') {sh 'podman rmi -i localhost/$IMAGE_NAME || true'}
}
}
}
Expand Down

0 comments on commit a495930

Please sign in to comment.