Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat/k8s #900

Draft
wants to merge 3 commits into
base: dev
Choose a base branch
from
Draft

Feat/k8s #900

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7d6420e
as-is working k8s deces-ui, deces-backend, redis, elasticseach
rhanka Mar 12, 2024
228b397
k8s: add ingress local - now fully functional
rhanka Mar 12, 2024
9f88d1c
k8s wip commit
rhanka Mar 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 39 additions & 16 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,11 +113,10 @@ export GIT_TOOLS = tools
export API_URL?=${APP_DNS}
export API_SSL?=1
export APP_NODES=1
# export KUBE_NAMESPACE:=$(shell echo -n ${APP_GROUP}-${APP}-${GIT_BRANCH} | tr '[:upper:]' '[:lower:]' | tr '_' '-')
export KUBE_NAMESPACE:=$(shell echo -n ${APP_GROUP}-${APP}-${GIT_BRANCH} | tr '[:upper:]' '[:lower:]' | tr '_/' '-')
export KUBE_DIR=${FRONTEND}/k8s
export ES_MEM_KUBE:=$(shell echo -n ${ES_MEM} | sed 's/\s*m/Mi/')
export STORAGE_ACCESS_KEY_B64:=$(shell echo -n ${STORAGE_ACCESS_KEY} | openssl base64)
export STORAGE_SECRET_KEY_B64:=$(shell echo -n ${STORAGE_SECRET_KEY} | openssl base64)
export KUBECONFIG=${HOME}/.kube/config
export ES_MEM_KUBE?=$(shell echo -n ${ES_MEM} | sed 's/\s*m/Mi/')

export PROOFS=${FRONTEND}/${GIT_BACKEND}/backend/data/proofs
export MONITOR_DIR = ${APP}/log/instances/${APP_GROUP}-${APP}-${GIT_BRANCH}
Expand Down Expand Up @@ -168,6 +167,9 @@ export SCW_IMAGE_ID=7a1d4022-766c-42b0-866e-d9c3cbd9d3b4
dummy := $(shell touch artifacts)
include ./artifacts

export STORAGE_ACCESS_KEY_B64:=$(shell echo -n ${STORAGE_ACCESS_KEY} | openssl base64)
export STORAGE_SECRET_KEY_B64:=$(shell echo -n ${STORAGE_SECRET_KEY} | openssl base64)

export VERSION := $(shell cat tagfiles.version | xargs -I '{}' find {} -type f -not -name '*.tar.gz' | sort | xargs cat | sha1sum - | sed 's/\(......\).*/\1/')

commit := $(shell git describe --tags || cat VERSION )
Expand Down Expand Up @@ -524,31 +526,51 @@ local-test-api:
done ; \
exit $$ret

deploy-k8s-cluster-local:
@if ! (which k3s > /dev/null 2>&1); then\
(curl -sfL https://get.k3s.io | sh - 2>&1 |\
awk 'BEGIN{s=0}{printf "\r☸️ Installing k3s (" s++ "/16)"}') && echo -e "\r\033[2K☸️ Installed k3s";\
fi;\
mkdir -p ~/.kube;\
KUBECONFIG=${HOME}/.kube/config-local-k3s.yaml;\
sudo cp /etc/rancher/k3s/k3s.yaml $${KUBECONFIG};\
sudo chown ${USER} $${KUBECONFIG};\
cp $${KUBECONFIG} ${KUBECONFIG}
Comment on lines +529 to +538
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The script for setting up a local Kubernetes cluster using k3s is well-structured. However, consider checking for existing ~/.kube/config-local-k3s.yaml before overwriting it to avoid accidental loss of existing configurations.

+ if [ ! -f "$${KUBECONFIG}" ]; then
    sudo cp /etc/rancher/k3s/k3s.yaml $${KUBECONFIG};
    sudo chown ${USER} $${KUBECONFIG};
    cp $${KUBECONFIG} ${KUBECONFIG}
+ fi
Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation.

Suggested change
deploy-k8s-cluster-local:
@if ! (which k3s > /dev/null 2>&1); then\
(curl -sfL https://get.k3s.io | sh - 2>&1 |\
awk 'BEGIN{s=0}{printf "\r☸️ Installing k3s (" s++ "/16)"}') && echo -e "\r\033[2K☸️ Installed k3s";\
fi;\
mkdir -p ~/.kube;\
KUBECONFIG=${HOME}/.kube/config-local-k3s.yaml;\
sudo cp /etc/rancher/k3s/k3s.yaml $${KUBECONFIG};\
sudo chown ${USER} $${KUBECONFIG};\
cp $${KUBECONFIG} ${KUBECONFIG}
deploy-k8s-cluster-local:
@if ! (which k3s > /dev/null 2>&1); then\
(curl -sfL https://get.k3s.io | sh - 2>&1 |\
awk 'BEGIN{s=0}{printf "\r☸️ Installing k3s (" s++ "/16)"}') && echo -e "\r\033[2K☸️ Installed k3s";\
fi;\
mkdir -p ~/.kube;\
KUBECONFIG=${HOME}/.kube/config-local-k3s.yaml;\
if [ ! -f "$${KUBECONFIG}" ]; then
sudo cp /etc/rancher/k3s/k3s.yaml $${KUBECONFIG};\
sudo chown ${USER} $${KUBECONFIG};\
cp $${KUBECONFIG} ${KUBECONFIG}
fi


deploy-k8s-local: deploy-k8s-services deploy-k8s-ingress-local


deploy-k8s: deploy-k8s-elasticsearch deploy-k8s-redis deploy-k8s-backend deploy-k8s-frontend
deploy-k8s-services: deploy-k8s-namespace deploy-k8s-elasticsearch deploy-k8s-redis deploy-k8s-backend deploy-k8s-frontend

deploy-k8s-namespace:
@echo $@
(cat ${KUBE_DIR}/namespace.yaml | envsubst | kubectl apply -f -) && touch $@
@echo $@;\
cat ${KUBE_DIR}/namespace.yaml | envsubst `env | sed "s/=.*//;s/^/$$/" | tr "\n" ","`;\
(cat ${KUBE_DIR}/namespace.yaml | envsubst `env | sed "s/=.*//;s/^/$$/" | tr "\n" ","` | kubectl apply -f -)

deploy-k8s-elasticsearch: deploy-k8s-namespace
deploy-k8s-elasticsearch: ${DATAPREP_VERSION_FILE} ${DATA_VERSION_FILE}
@echo $@
@cat ${KUBE_DIR}/elasticsearch.yaml | envsubst | kubectl apply -f -
@DATAPREP_VERSION=$$(cat ${DATAPREP_VERSION_FILE});\
DATA_VERSION=$$(cat ${DATA_VERSION_FILE});\
export ES_BACKUP_NAME=${ES_BACKUP_BASENAME}_$${DATAPREP_VERSION}_$${DATA_VERSION};\
echo SCW_REGION=${SCW_REGION} SCW_ENDPOINT=${SCW_ENDPOINT} SCW_BUCKET=${REPOSITORY_BUCKET};\
cat ${KUBE_DIR}/elasticsearch.yaml | envsubst `env | sed "s/=.*//;s/^/$$/" | tr "\n" ","` | kubectl apply -f -

deploy-k8s-redis: deploy-k8s-namespace
deploy-k8s-redis:
@echo $@
@cat ${KUBE_DIR}/redis.yaml | envsubst | kubectl apply -f -
@cat ${KUBE_DIR}/redis.yaml | envsubst `env | sed "s/=.*//;s/^/$$/" | tr "\n" ","` | kubectl apply -f -

deploy-k8s-backend: deploy-k8s-namespace
deploy-k8s-backend:
@echo $@
@export BACKEND_APP_VERSION=$(shell cd ${APP_PATH}/${GIT_BACKEND} && git describe --tags);\
cat ${KUBE_DIR}/backend.yaml | envsubst | kubectl apply -f -
cat ${KUBE_DIR}/backend.yaml | envsubst `env | sed "s/=.*//;s/^/$$/" | tr "\n" ","` | kubectl apply -f -

deploy-k8s-frontend: deploy-k8s-namespace
deploy-k8s-frontend:
@echo $@
@cat ${KUBE_DIR}/frontend.yaml | envsubst | kubectl apply -f -
@cat ${KUBE_DIR}/frontend.yaml | envsubst `env | sed "s/=.*//;s/^/$$/" | tr "\n" ","` | kubectl apply -f -

deploy-k8s-ingress-local:
@echo $@
@export APP_DNS=deces.matchid.local;\
cat ${KUBE_DIR}/ingress-local.yaml | envsubst `env | sed "s/=.*//;s/^/$$/" | tr "\n" ","` | kubectl apply -f -

deploy-remote-instance: config-minimal backend-config ${DATAPREP_VERSION_FILE} ${DATA_VERSION_FILE}
@\
Expand Down Expand Up @@ -622,6 +644,7 @@ deploy-docker-pull-base: deploy-remote-instance
@${MAKE} -C ${APP_PATH}/${GIT_TOOLS} remote-docker-pull DOCKER_IMAGE=docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION}
@${MAKE} -C ${APP_PATH}/${GIT_TOOLS} remote-docker-pull DOCKER_IMAGE=redis:alpine


update-base-image: deploy-remote-instance deploy-docker-pull-base
@BACKEND_APP_VERSION=$(shell cd ${APP_PATH}/${GIT_BACKEND} && git describe --tags); \
${MAKE} -C ${APP_PATH}/${GIT_TOOLS} remote-cmd REMOTE_CMD="sync"; \
Expand Down
208 changes: 168 additions & 40 deletions k8s/elasticsearch.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,67 +11,180 @@ data:
apiVersion: apps/v1
kind: Deployment
metadata:
name: ${APP}-es-deployment
name: elasticsearch-deployment
namespace: ${KUBE_NAMESPACE}
spec:
selector:
matchLabels:
app: ${APP}-es
app: elasticsearch
template:
metadata:
labels:
app: ${APP}-es
app: elasticsearch
spec:
initContainers:
- name: sysctl
image: docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION}
securityContext:
privileged: true
command: ['sh', '-c', 'sysctl -w vm.max_map_count=262144']
- name: install-plugins
image: docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION}
runAsUser: 0
command:
- sh
- -c
- |
bin/elasticsearch-plugin install --batch repository-s3
- name: add-aws-keys
- sh
- -c
- |
sysctl -w vm.max_map_count=262144;
sysctl -w fs.file-max=65536;
echo Set vm.max_map_count and fs.file-max with success
# - name: add-aws-keys
# image: docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION}
# env:
# - name: AWS_ACCESS_KEY_ID
# valueFrom:
# secretKeyRef:
# name: s3-data-keys
# key: AWS_ACCESS_KEY_ID
# - name: AWS_SECRET_ACCESS_KEY
# valueFrom:
# secretKeyRef:
# name: s3-data-keys
# key: AWS_SECRET_ACCESS_KEY
# command:
# - sh
# - -c
# - |
# echo Adding $AWS_ACCESS_KEY_ID;
# echo $AWS_ACCESS_KEY_ID | bin/elasticsearch-keystore add --stdin --force s3.client.default.access_key;
# echo $AWS_SECRET_ACCESS_KEY | bin/elasticsearch-keystore add --stdin --force s3.client.default.secret_key;
# echo Added AWS keys with success
- name: restore-snapshot
image: docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION}
volumeMounts:
- name: elasticsearch-data
mountPath: /usr/share/elasticsearch/data
env:
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: s3-data-keys
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: s3-data-keys
key: AWS_SECRET_ACCESS_KEY
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: s3-data-keys
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: s3-data-keys
key: AWS_SECRET_ACCESS_KEY
- name: xpack.security.enabled
value: "false"
- name: ES_JAVA_OPTS
value: -Xms${ES_MEM} -Xmx${ES_MEM}
- name: discovery.type
value: single-node
- name: http_proxy
value: ${http_proxy}
- name: https_proxy
value: ${https_proxy}
- name: no_proxy
value: ${no_proxy}
- name: LOG4J2_FORMAT_MSG_NO_LOOKUPS
value: "true"
- name: bootstrap.memory_lock
value: "true"
- name: REPOSITORY_BUCKET
value: ${REPOSITORY_BUCKET}
- name: SCW_REGION
value: ${SCW_REGION}
- name: SCW_ENDPOINT
value: ${SCW_ENDPOINT}
- name: APP_GROUP
value: ${APP_GROUP}
- name: ES_BACKUP_NAME
value: ${ES_BACKUP_NAME}
- name: ES_INDEX
value: ${ES_INDEX}
- name: ES_TIMEOUT
value: "${ES_TIMEOUT}"
- name: ES_RESTORE_TIMEOUT
value: "${ES_RESTORE_TIMEOUT}"
command:
- sh
- -c
- |
echo $AWS_ACCESS_KEY_ID | bin/elasticsearch-keystore add --stdin --force s3.client.default.access_key
echo $AWS_SECRET_ACCESS_KEY | bin/elasticsearch-keystore add --stdin --force s3.client.default.secret_key
- sh
- -c
- |
echo $AWS_ACCESS_KEY_ID | bin/elasticsearch-keystore add --stdin --force s3.client.default.access_key;
echo $AWS_SECRET_ACCESS_KEY | bin/elasticsearch-keystore add --stdin --force s3.client.default.secret_key;
PID=$(/usr/local/bin/docker-entrypoint.sh > /dev/null & echo $!);
timeout=${ES_TIMEOUT};
ret=1 ;
until [ "$timeout" -le 0 -o "$ret" -eq "0" ] ;
do (curl -s --fail --connect-timeout 1 --max-time 1 -XGET localhost:9200/ > /dev/null) ;
ret=$? ;
if [ "$ret" -ne "0" ] ;
then echo "Waiting for elasticsearch to start $timeout" ;
fi ;
((timeout--));
sleep 1 ;
done ;
if [ "$ret" -ne "0" ] ;
then echo "Failed to start elasticsearch" && exit 1 ;
fi ;
echo "Adding repository ${APP_GROUP} from S3 bucket ${REPOSITORY_BUCKET}";
curl -s -XPUT "localhost:9200/_snapshot/${APP_GROUP}" -H 'Content-Type: application/json' -d '{"type": "s3","settings": {"bucket": "${REPOSITORY_BUCKET}","client": "default","region": "${SCW_REGION}","endpoint": "${SCW_ENDPOINT}","path_style_access": true,"protocol": "https"}}' > /tmp/repo.json ;
cat /tmp/repo.json;
cat /tmp/repo.json | grep -q '"acknowledged":true' || (echo "\nFailed" && exit 1);
echo -e "\nList of available snapshots in S3 bucket ${REPOSITORY_BUCKET}";
curl -s -XGET localhost:9200/_snapshot/${APP_GROUP}/_all ;
echo -e "\nRestoring snapshot ${ES_BACKUP_NAME} from S3 bucket";
curl -s -XPOST localhost:9200/_snapshot/${APP_GROUP}/${ES_BACKUP_NAME}/_restore -H 'Content-Type: application/json' -d '{"indices": "${ES_INDEX}","ignore_unavailable": true,"include_global_state": false}' > /tmp/snapshot.json ;
cat /tmp/snapshot.json;
cat /tmp/snapshot.json | grep -q '"accepted":true' || (echo -e "\nFailed" && exit 1);
echo;
timeout=${ES_RESTORE_TIMEOUT} ;
ret=1 ; until [ "$timeout" -le 0 -o "$ret" -eq "0" ] ;
do (curl -s --fail -XGET localhost:9200/_cat/indices | grep ${ES_INDEX} | grep -q green > /dev/null) ;
ret=$? ;
if [ "$$ret" -ne "0" ] ; then
INDEX_SIZE=$(du -sh /usr/share/elasticsearch/data | awk '{print $1}');
echo "Waiting for elasticsearch index ${ES_INDEX} (${INDEX_SIZE}) to be green $timeout" ;
fi ;
((timeout--));
sleep 1 ;
done ;
if [ "$ret" -ne "0" ] ;
then echo "Failed to restore snapshot" && exit 1 ;
fi ;
echo "Restored snapshot with success";
echo "Terminating elasticsearch process PID ${PID}";
kill ${PID};
rm -f /usr/share/elasticsearch/data/**/node.lock;
containers:
- name: elasticsearch
image: docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION}
ports:
- containerPort: ${ES_PORT}
env:
- name: ES_JAVA_OPTS
value: -Xms${ES_MEM} -Xmx${ES_MEM}
- name: discovery.type
value: single-node
resources:
requests:
memory: ${ES_MEM_KUBE}
- name: elasticsearch
image: docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION}
ports:
- containerPort: ${ES_PORT}
env:
- name: ES_JAVA_OPTS
value: -Xms${ES_MEM} -Xmx${ES_MEM}
- name: discovery.type
value: single-node
- name: xpack.security.enabled
value: "false"
- name: LOG4J2_FORMAT_MSG_NO_LOOKUPS
value: "true"
- name: bootstrap.memory_lock
value: "true"
resources:
requests:
memory: ${ES_MEM_KUBE}
volumeMounts:
- name: elasticsearch-data
mountPath: /usr/share/elasticsearch/data
volumes:
- name: elasticsearch-data
persistentVolumeClaim:
claimName: elasticsearch-data
---
apiVersion: v1
kind: Service
metadata:
name: ${APP}-es-svc
name: elasticsearch
namespace: ${KUBE_NAMESPACE}
spec:
ports:
Expand All @@ -80,4 +193,19 @@ spec:
protocol: TCP
name: http
selector:
app: ${APP}-es
app: elasticsearch
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: elasticsearch-data
namespace: ${KUBE_NAMESPACE}
labels:
app: elasticsearch
spec:
storageClassName: local-path
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 30Gi
8 changes: 5 additions & 3 deletions k8s/frontend.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,16 @@ spec:
- name: ${DC_PREFIX}
image: ${DOCKER_USERNAME}/${DC_PREFIX}:${APP_VERSION}
ports:
- containerPort: 80
- containerPort: ${PORT}
env:
- name: PORT
value: "${PORT}"
- name: APP
value: "${APP}"
- name: ES_PROXY_PATH
value: "${ES_PROXY_PATH}"
- name: ES_HOST
value: "${APP}-es-svc"
value: "elasticsearch"
- name: ES_INDEX
value: "${ES_INDEX}"
- name: ES_PORT
Expand Down Expand Up @@ -94,7 +96,7 @@ metadata:
spec:
ports:
- port: 80
targetPort: 80
targetPort: ${PORT}
protocol: TCP
name: ui
selector:
Expand Down
17 changes: 17 additions & 0 deletions k8s/ingress-local.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ${APP}-ingress
namespace: ${KUBE_NAMESPACE}
spec:
rules:
- host: ${APP_DNS}
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: ${APP}-svc
port:
number: 80
Loading