diff --git a/Jenkinsfile b/Jenkinsfile index c3808daf05..f7361eb6da 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -206,7 +206,7 @@ pipeline { stage('docsite') { steps { - sh "${RunInBuilder()} ${env.BUILD_CONTAINER} make vhtml" + sh "${RunInBuilder()} ${env.BUILD_CONTAINER} env GITHUB_SHA=${GIT_COMMIT} GITHUB_REPOSITORY=${SourceRepo()} make vhtml" publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: false, reportDir: '_output/html', reportFiles: 'index.html', reportName: 'Doc Site', reportTitles: '']) } } @@ -382,6 +382,18 @@ String RunInBuilder() { " } +/* + Returns / from which the code was built. +*/ +String SourceRepo() { + // Content of CHANGE_FORK varies, see https://issues.jenkins-ci.org/browse/JENKINS-58450. + (! env.CHANGE_FORK) ? + "github.com/intel/pmem-csi" : + env.CHANGE_FORK.matches('.*/.*') ? + env.CHANGE_FORK : + env.CHANGE_FORK + '/pmem-csi' +} + void TestInVM(distro, distroVersion, kubernetesVersion, skipIfPR) { try { /* diff --git a/Makefile b/Makefile index ba513c0d60..b2a5481e90 100644 --- a/Makefile +++ b/Makefile @@ -233,14 +233,19 @@ SOURCEDIR = . BUILDDIR = _output # Generate doc site under _build/html with Sphinx. +# "vhtml" will set up tools, "html" expects them to be installed. +# GITHUB_SHA will be used for kustomize references to the GitHub +# repo (= github.com/intel/pmem-csi/deploy, a syntax that is only +# valid there) if set. +GEN_DOCS = $(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) && \ + ( ! [ "$$GITHUB_SHA" ] || ! [ "$$GITHUB_REPOSITORY" ] || \ + find $(BUILDDIR)/html/ -name '*.html' | \ + xargs sed -i -e "s;github.com/intel/pmem-csi/\\(deploy/\\S*\\);github.com/$$GITHUB_REPOSITORY/\\1?ref=$$GITHUB_SHA;g" ) && \ + cp docs/html/index.html $(BUILDDIR)/html/index.html vhtml: _work/venv/.stamp - . _work/venv/bin/activate && \ - $(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) && \ - cp docs/html/index.html $(BUILDDIR)/html/index.html - + . _work/venv/bin/activate && $(GEN_DOCS) html: - $(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) && \ - cp docs/html/index.html $(BUILDDIR)/html/index.html + $(GEN_DOCS) clean-html: rm -rf _output/html diff --git a/conf.py b/conf.py index ca406bfc34..d83dbd5b6f 100644 --- a/conf.py +++ b/conf.py @@ -29,7 +29,7 @@ baseBranch = "devel" useGitHubURL = True commitSHA = getenv('GITHUB_SHA') -githubBaseURL = "https://github.com/intel/pmem-csi/" +githubBaseURL = 'https://github.com/' + (getenv('GITHUB_REPOSITORY') or 'intel/pmem-csi') + '/' githubFileURL = githubBaseURL + "blob/" githubDirURL = githubBaseURL + "tree/" if commitSHA: diff --git a/deploy/kustomize/memcached/ephemeral/kustomization.yaml b/deploy/kustomize/memcached/ephemeral/kustomization.yaml new file mode 100644 index 0000000000..c38a75b98d --- /dev/null +++ b/deploy/kustomize/memcached/ephemeral/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - memcached-ephemeral.yaml diff --git a/deploy/kustomize/memcached/ephemeral/memcached-ephemeral.yaml b/deploy/kustomize/memcached/ephemeral/memcached-ephemeral.yaml new file mode 100644 index 0000000000..7e9249d965 --- /dev/null +++ b/deploy/kustomize/memcached/ephemeral/memcached-ephemeral.yaml @@ -0,0 +1,87 @@ +apiVersion: v1 +kind: Service +metadata: + name: pmem-memcached + namespace: default +spec: + ports: + - name: db + port: 11211 + protocol: TCP + targetPort: db + selector: + app.kubernetes.io/kind: memcached + app.kubernetes.io/name: pmem-memcached + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pmem-memcached + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/kind: memcached + app.kubernetes.io/name: pmem-memcached + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app.kubernetes.io/kind: memcached + app.kubernetes.io/name: pmem-memcached + spec: + initContainers: + - name: data-volume-init + # This first creates a file that is as large as root can make it (larger + # than a normal user because of reserved blocks). Size is rounded down to + # MiB because that is what memcached expects and 50 MiB are substracted + # for filesystem overhead. + # + # Then it changes the ownership of the data volume so that the memcached user + # can read and write files there. The exact UID varies between + # memcached image versions (therefore we cannot use Kubernetes to change + # the ownership for us as that relies on a numeric value), but the "memcache" + # name is the same, so by running in the same image as memcached we can set + # the owner by name. + command: + - sh + - -c + - | + fallocate --length=$(( $(stat --file-system --format='%b * %s / 1024 / 1024 - 50' /data) ))MiB /data/memcached-memory-file && + chown -R memcache /data + image: memcached:1.5.22 + securityContext: + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /data + name: data-volume + containers: + - name: memcached + image: memcached:1.5.22 + command: + - sh + - -c + - exec memcached --memory-limit=$(( $(stat --format='%s / 1024 / 1024' /data/memcached-memory-file) )) --memory-file=/data/memcached-memory-file $(MEMCACHED_ARGS) + ports: + - containerPort: 11211 + name: db + protocol: TCP + volumeMounts: + - mountPath: /data + name: data-volume + env: + - name: MEMCACHED_ARGS + value: + volumes: + - name: data-volume + csi: + driver: pmem-csi.intel.com + volumeAttributes: + size: 200Mi diff --git a/deploy/kustomize/memcached/persistent/kustomization.yaml b/deploy/kustomize/memcached/persistent/kustomization.yaml new file mode 100644 index 0000000000..e15ec256a2 --- /dev/null +++ b/deploy/kustomize/memcached/persistent/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - memcached-persistent.yaml diff --git a/deploy/kustomize/memcached/persistent/memcached-persistent.yaml b/deploy/kustomize/memcached/persistent/memcached-persistent.yaml new file mode 100644 index 0000000000..765ce7abe8 --- /dev/null +++ b/deploy/kustomize/memcached/persistent/memcached-persistent.yaml @@ -0,0 +1,92 @@ +apiVersion: v1 +kind: Service +metadata: + name: pmem-memcached + namespace: default +spec: + ports: + - name: db + port: 11211 + protocol: TCP + targetPort: db + selector: + app.kubernetes.io/kind: memcached + app.kubernetes.io/name: pmem-memcached + type: ClusterIP +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: pmem-memcached + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/kind: memcached + app.kubernetes.io/name: pmem-memcached + serviceName: pmem-memcached + template: + metadata: + labels: + app.kubernetes.io/kind: memcached + app.kubernetes.io/name: pmem-memcached + spec: + initContainers: + - name: data-volume-init + # This first creates a file that is as large as root can make it (larger + # than a normal user because of reserved blocks). Size is rounded down to + # MiB because that is what memcached expects and 50 MiB are substracted + # for filesystem overhead and the memcached state files. + # + # Then it changes the ownership of the data volume so that the memcached user + # can read and write files there. The exact UID varies between + # memcached image versions (therefore we cannot use Kubernetes to change + # the ownership for us as that relies on a numeric value), but the "memcache" + # name is the same, so by running in the same image as memcached we can set + # the owner by name. + command: + - sh + - -c + - | + fallocate --length=$(( $(stat --file-system --format='%b * %s / 1024 / 1024 - 50' /data) ))MiB /data/memcached-memory-file && + chown -R memcache /data + image: memcached:1.5.22 + securityContext: + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /data + name: memcached-data-volume + containers: + - name: memcached + image: memcached:1.5.22 + command: + - sh + - -c + - | + set -x + memcached --memory-limit=$(( $(stat --format='%s / 1024 / 1024' /data/memcached-memory-file) )) --memory-file=/data/memcached-memory-file $(MEMCACHED_ARGS) & + pid=$$! + trap 'kill -USR1 $$pid; wait $$pid' TERM + wait $$pid + ports: + - containerPort: 11211 + name: db + protocol: TCP + volumeMounts: + - mountPath: /data + name: memcached-data-volume + env: + - name: MEMCACHED_ARGS + value: + terminationGracePeriodSeconds: 30 + volumeClaimTemplates: + - metadata: + name: memcached-data-volume + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: pmem-csi-sc-ext4 + resources: + requests: + storage: 200Mi diff --git a/examples/memcached.md b/examples/memcached.md new file mode 100644 index 0000000000..ed091bb2a7 --- /dev/null +++ b/examples/memcached.md @@ -0,0 +1,307 @@ +# memcached + +As shown in [this blog +post](https://memcached.org/blog/persistent-memory/), memcached can +efficiently utilize PMEM as replacement for DRAM. This makes it +possible to increase cache sizes and/or reduce costs. The memcached +maintainers make container images available which support this +feature. This example shows how to take advantage of PMEM with +memcached running on Kubernetes. + +## Prerequisites + +The instructions below assume that: +- `kubectl` is available and can access the cluster. +- PMEM-CSI [was installed](/docs/install.md#installation-and-setup) + with the default `pmem-csi.intel.com` driver name and with the + [`pmem-csi-sc-ext4` storage + class](/deploy/common/pmem-storageclass-ext4.yaml). +- `telnet` or similar tools like `socat` are available. + +It is not necessary to check out the PMEM-CSI source code. + +## PMEM as DRAM replacement + +Memcached can map PMEM into its address space and then store the bulk +of the cache there. The advantage is higher capacity and (depending on +pricing) less costs. + +The cache could be stored in a persistent volume and reused after a +restart (see next section), but when that isn't the goal, a +[deployment with one CSI ephemeral inline +volume](/deploy/kustomize/memcached/ephemeral/memcached-ephemeral.yaml) +per memcached pod is very simple: +- A Kubernetes deployment manages the memcached instance(s). +- In the pod template, one additional PMEM volume of a certain size is + requested. +- An init container prepares that volume for use by memcached: + - It determines how large the actual file can be that memcached will use. + - It changes volume ownership so that memcached can run as non-root + process. +- A wrapper script adds the `--memory-file` and `--memory-limit` parameters + when invoking memcached, which enables the use of PMEM. + +In this mode, the PMEM-CSI driver is referenced through its name +(usually `pmem-csi.intel.com`). No storage classes are needed. That +name and other parameters in the deployment can be modified with +[`kustomize`](https://github.com/kubernetes-sigs/kustomize). Here's +how one can change the namespace, volume size or add additional +command line parameters: + +```console +$ mkdir -p my-memcached-deployment + +$ cat >my-memcached-deployment/kustomization.yaml <my-memcached-deployment/update-deployment.yaml < 11211 +Forwarding from [::1]:11211 -> 11211 +``` + +In another shell we can now use `telnet` to connect to memcached: +```console +$ telnet localhost 11211 +Trying ::1... +Connected to localhost. +Escape character is '^]'. +``` + +Memcached accepts simple plain-text commands. To set a key with 10 +characters, a time-to-live of 500 seconds and default flags, enter: +``` +set demo_key 0 500000 10 +I am PMEM. +``` + +Memcached acknowledges this with: +``` +STORED +``` + +We can verify that the key exists with: +``` +get demo_key +``` + +``` +VALUE demo_key 0 10 +I am PMEM. +END +``` + +To disconnect, use: +``` +quit +``` + +``` +Connection closed by foreign host. +``` + +The following command verifies the data was stored in a persistent +memory data volume: +```console +$ kubectl exec -n demo $(kubectl get -n demo pods -l app.kubernetes.io/name=pmem-memcached -o jsonpath={..metadata.name}) grep 'I am PMEM.' /data/memcached-memory-file +Binary file /data/memcached-memory-file matches +``` + +To clean up, terminate the `kubectl port-forward` command and delete the memcached deployment with: +```console +$ kubectl delete --kustomize my-memcached-deployment +service "pmem-memcached" deleted +deployment.apps "pmem-memcached" deleted +``` + + + +## Restartable Cache + +The [restartable cache +feature](https://github.com/memcached/memcached/wiki/WarmRestart) +works like non-persistent usage of PMEM. In addition, memcached writes +out one additional, short state file during a shutdown triggered by +`SIGUSR1`. The state file describes the content of the memory file and +is used during a restart by memcached to decide whether it can use the +existing cached data. + +The [example +deployment](/deploy/kustomize/memcached/persistent/memcached-persistent.yaml) +uses a stateful set because that can automatically create persistent +volumes for each instance. The shell wrapper around memcached +translates the normal [`SIGTERM` shutdown +signal](https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods) +into `SIGUSR1`. + +Deploying like that becomes less flexible because the memcached pods +can no longer move freely between nodes. Instead, each pod has to be +restarted on the node where its volume was created. There are also +[several caveats for memcached in this +mode](https://github.com/memcached/memcached/wiki/WarmRestart#caveats) +that admins and application developers must be aware of. + +This example can also be kustomized. It uses the [`pmem-csi-sc-ext4` +storage class](/deploy/common/pmem-storageclass-ext4.yaml). Here we +just use the defaults, in particular the default namespace: + +```console +$ kubectl apply --kustomize github.com/intel/pmem-csi/deploy/kustomize/memcached/persistent +service/pmem-memcached created +statefulset.apps/pmem-memcached created +``` + +We can verify that memcached really does a warm restart by storing +some data, removing the instance and then starting it again. + +```console +$ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/name=pmem-memcached +pod/pmem-memcached-0 condition met +``` + +Because we use a stateful set, the pod name is deterministic. + +There is also a corresponding persistent volume: +```console +$ kubectl get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +memcached-data-volume-pmem-memcached-0 Bound pvc-bb2cde11-6aa2-46da-8521-9bc35c08426d 200Mi RWO pmem-csi-sc-ext4 5m45s +``` + +First set a key using the same approach as before: +```console +$ kubectl port-forward service/pmem-memcached 11211 +Forwarding from 127.0.0.1:11211 -> 11211 +Forwarding from [::1]:11211 -> 11211 +``` + +```console +$ telnet localhost 11211 +Trying ::1... +Connected to localhost. +Escape character is '^]'. +``` + +``` +set demo_key 0 500000 10 +I am PMEM. +``` + +``` +STORED +``` + +``` +quit +``` + +Then scale down the number of memcached instances down to zero, then +restart it. To avoid race conditions, it is important to wait for +Kubernetes to catch up: + +```console +$ kubectl scale --replicas=0 statefulset/pmem-memcached +statefulset.apps/pmem-memcached scaled + +$ kubectl wait --for delete pod/pmem-memcached-0 +Error from server (NotFound): pods "pmem-memcached-0" not found + +$ kubectl scale --replicas=1 statefulset/pmem-memcached +statefulset.apps/pmem-memcached scaled + +$ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/name=pmem-memcached +pod/pmem-memcached-0 condition met +``` + +Restart the port forwarding now because it is tied to the previous pod. + +Without the persistent volume and the restartable cache, the memcached +cache would be empty now. With `telnet` we can verify that this is not +the case and that the key is still known: +```console +$ telnet 127.0.0.1 11211 +Trying 127.0.0.1... +Connected to 127.0.0.1. +Escape character is '^]'. +``` + +``` +get demo_key +``` + +``` +VALUE demo_key 0 10 +I am PMEM. +END +``` + +``` +quit +``` + +``` +Connection closed by foreign host. +``` + +To clean up, terminate the `kubectl port-forward` command and delete the memcached deployment with: +```console +$ kubectl delete --kustomize github.com/intel/pmem-csi/deploy/kustomize/memcached/persistent +service "pmem-memcached" deleted +statefulset.apps "pmem-memcached" deleted +``` + +Beware that at the moment, the volumes need to be removed manually +after removing the stateful set. A [request to automate +that](https://github.com/kubernetes/kubernetes/issues/55045) is open. + +```console +$ kubectl delete pvc -l app.kubernetes.io/name=pmem-memcached +persistentvolumeclaim "memcached-data-volume-pmem-memcached-0" deleted +``` diff --git a/examples/readme.rst b/examples/readme.rst index 72db86de6c..f99c21c0c0 100644 --- a/examples/readme.rst +++ b/examples/readme.rst @@ -4,6 +4,9 @@ Application examples `Redis-pmem operator `__ Deploy a Redis cluster through the redis-operator using QEMU-emulated persistent memory devices +`memcached with PMEM `__ + Deploy memcached with PMEM as replacement for DRAM. + `Google Cloud Engine `__ Install Kubernetes and PMEM-CSI on Google Cloud machines. @@ -11,5 +14,6 @@ Application examples :hidden: redis-operator.md + memcached.md gce.md - \ No newline at end of file +