Merge pull request #633 from pohly/memcached

examples: memcached with and without warm restart
intel · May 27, 2020 · cb9b36b · cb9b36b
2 parents 7d84e1a + 2b799d6
commit cb9b36b
Show file tree

Hide file tree

Showing 9 changed files with 520 additions and 9 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -206,7 +206,7 @@ pipeline {
 
         stage('docsite') {
             steps {
-                sh "${RunInBuilder()} ${env.BUILD_CONTAINER} make vhtml"
+                sh "${RunInBuilder()} ${env.BUILD_CONTAINER} env GITHUB_SHA=${GIT_COMMIT} GITHUB_REPOSITORY=${SourceRepo()} make vhtml"
                 publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: false, reportDir: '_output/html', reportFiles: 'index.html', reportName: 'Doc Site', reportTitles: ''])
             }
         }
@@ -382,6 +382,18 @@ String RunInBuilder() {
     "
 }
 
+/*
+ Returns <owner>/<repo> from which the code was built.
+*/
+String SourceRepo() {
+    // Content of CHANGE_FORK varies, see https://issues.jenkins-ci.org/browse/JENKINS-58450.
+    (! env.CHANGE_FORK) ?
+        "github.com/intel/pmem-csi" :
+    env.CHANGE_FORK.matches('.*/.*') ?
+        env.CHANGE_FORK :
+        env.CHANGE_FORK + '/pmem-csi'
+}
+
 void TestInVM(distro, distroVersion, kubernetesVersion, skipIfPR) {
     try {
         /*

diff --git a/Makefile b/Makefile
@@ -233,14 +233,19 @@ SOURCEDIR     = .
 BUILDDIR      = _output
 
 # Generate doc site under _build/html with Sphinx.
+# "vhtml" will set up tools, "html" expects them to be installed.
+# GITHUB_SHA will be used for kustomize references to the GitHub
+# repo (= github.com/intel/pmem-csi/deploy, a syntax that is only
+# valid there) if set.
+GEN_DOCS = $(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) && \
+	( ! [ "$$GITHUB_SHA" ] || ! [ "$$GITHUB_REPOSITORY" ] || \
+	  find $(BUILDDIR)/html/ -name '*.html' | \
+	  xargs sed -i -e "s;github.com/intel/pmem-csi/\\(deploy/\\S*\\);github.com/$$GITHUB_REPOSITORY/\\1?ref=$$GITHUB_SHA;g" ) && \
+	cp docs/html/index.html $(BUILDDIR)/html/index.html
 vhtml: _work/venv/.stamp
-	. _work/venv/bin/activate && \
-		$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) && \
-		cp docs/html/index.html $(BUILDDIR)/html/index.html
-
+	. _work/venv/bin/activate && $(GEN_DOCS)
 html:
-		$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) && \
-		cp docs/html/index.html $(BUILDDIR)/html/index.html
+	$(GEN_DOCS)
 
 clean-html:
 	rm -rf _output/html

diff --git a/conf.py b/conf.py
@@ -29,7 +29,7 @@
 baseBranch = "devel"
 useGitHubURL = True
 commitSHA = getenv('GITHUB_SHA')
-githubBaseURL = "https://github.com/intel/pmem-csi/"
+githubBaseURL = 'https://github.com/' + (getenv('GITHUB_REPOSITORY') or 'intel/pmem-csi') + '/'
 githubFileURL = githubBaseURL + "blob/"
 githubDirURL = githubBaseURL + "tree/"
 if commitSHA:

diff --git a/deploy/kustomize/memcached/ephemeral/kustomization.yaml b/deploy/kustomize/memcached/ephemeral/kustomization.yaml
@@ -0,0 +1,2 @@
+resources:
+  - memcached-ephemeral.yaml
diff --git a/deploy/kustomize/memcached/ephemeral/memcached-ephemeral.yaml b/deploy/kustomize/memcached/ephemeral/memcached-ephemeral.yaml
@@ -0,0 +1,87 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: pmem-memcached
+  namespace: default
+spec:
+  ports:
+  - name: db
+    port: 11211
+    protocol: TCP
+    targetPort: db
+  selector:
+    app.kubernetes.io/kind: memcached
+    app.kubernetes.io/name: pmem-memcached
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: pmem-memcached
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/kind: memcached
+      app.kubernetes.io/name: pmem-memcached
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/kind: memcached
+        app.kubernetes.io/name: pmem-memcached
+    spec:
+      initContainers:
+      - name: data-volume-init
+        # This first creates a file that is as large as root can make it (larger
+        # than a normal user because of reserved blocks). Size is rounded down to
+        # MiB because that is what memcached expects and 50 MiB are substracted
+        # for filesystem overhead.
+        #
+        # Then it changes the ownership of the data volume so that the memcached user
+        # can read and write files there. The exact UID varies between
+        # memcached image versions (therefore we cannot use Kubernetes to change
+        # the ownership for us as that relies on a numeric value), but the "memcache"
+        # name is the same, so by running in the same image as memcached we can set
+        # the owner by name.
+        command:
+        - sh
+        - -c
+        - |
+          fallocate --length=$(( $(stat --file-system --format='%b * %s / 1024 / 1024 - 50' /data) ))MiB /data/memcached-memory-file &&
+          chown -R memcache /data
+        image: memcached:1.5.22
+        securityContext:
+          privileged: true
+          runAsUser: 0
+        volumeMounts:
+        - mountPath: /data
+          name: data-volume
+      containers:
+      - name: memcached
+        image: memcached:1.5.22
+        command:
+        - sh
+        - -c
+        - exec memcached --memory-limit=$(( $(stat --format='%s / 1024 / 1024' /data/memcached-memory-file) )) --memory-file=/data/memcached-memory-file $(MEMCACHED_ARGS)
+        ports:
+        - containerPort: 11211
+          name: db
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /data
+          name: data-volume
+        env:
+        - name: MEMCACHED_ARGS
+          value:
+      volumes:
+      - name: data-volume
+        csi:
+          driver: pmem-csi.intel.com
+          volumeAttributes:
+            size: 200Mi
diff --git a/deploy/kustomize/memcached/persistent/kustomization.yaml b/deploy/kustomize/memcached/persistent/kustomization.yaml
@@ -0,0 +1,2 @@
+resources:
+  - memcached-persistent.yaml
diff --git a/deploy/kustomize/memcached/persistent/memcached-persistent.yaml b/deploy/kustomize/memcached/persistent/memcached-persistent.yaml
@@ -0,0 +1,92 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: pmem-memcached
+  namespace: default
+spec:
+  ports:
+  - name: db
+    port: 11211
+    protocol: TCP
+    targetPort: db
+  selector:
+    app.kubernetes.io/kind: memcached
+    app.kubernetes.io/name: pmem-memcached
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: pmem-memcached
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/kind: memcached
+      app.kubernetes.io/name: pmem-memcached
+  serviceName: pmem-memcached
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/kind: memcached
+        app.kubernetes.io/name: pmem-memcached
+    spec:
+      initContainers:
+      - name: data-volume-init
+        # This first creates a file that is as large as root can make it (larger
+        # than a normal user because of reserved blocks). Size is rounded down to
+        # MiB because that is what memcached expects and 50 MiB are substracted
+        # for filesystem overhead and the memcached state files.
+        #
+        # Then it changes the ownership of the data volume so that the memcached user
+        # can read and write files there. The exact UID varies between
+        # memcached image versions (therefore we cannot use Kubernetes to change
+        # the ownership for us as that relies on a numeric value), but the "memcache"
+        # name is the same, so by running in the same image as memcached we can set
+        # the owner by name.
+        command:
+        - sh
+        - -c
+        - |
+          fallocate --length=$(( $(stat --file-system --format='%b * %s / 1024 / 1024 - 50' /data) ))MiB /data/memcached-memory-file &&
+          chown -R memcache /data
+        image: memcached:1.5.22
+        securityContext:
+          privileged: true
+          runAsUser: 0
+        volumeMounts:
+        - mountPath: /data
+          name: memcached-data-volume
+      containers:
+      - name: memcached
+        image: memcached:1.5.22
+        command:
+        - sh
+        - -c
+        - |
+          set -x
+          memcached --memory-limit=$(( $(stat --format='%s / 1024 / 1024' /data/memcached-memory-file) )) --memory-file=/data/memcached-memory-file $(MEMCACHED_ARGS) &
+          pid=$$!
+          trap 'kill -USR1 $$pid; wait $$pid' TERM
+          wait $$pid
+        ports:
+        - containerPort: 11211
+          name: db
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /data
+          name: memcached-data-volume
+        env:
+        - name: MEMCACHED_ARGS
+          value:
+      terminationGracePeriodSeconds: 30
+  volumeClaimTemplates:
+    - metadata:
+        name: memcached-data-volume
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        storageClassName: pmem-csi-sc-ext4
+        resources:
+          requests:
+            storage: 200Mi