diff --git a/samples/long-running-compute/jmeter.log b/samples/long-running-compute/jmeter.log new file mode 100644 index 0000000..eb64cfc --- /dev/null +++ b/samples/long-running-compute/jmeter.log @@ -0,0 +1,63 @@ +2020-10-16 21:40:47,200 INFO o.a.j.u.JMeterUtils: Setting Locale to en_EN +2020-10-16 21:40:47,247 INFO o.a.j.JMeter: Loading user properties from: /usr/local/Cellar/jmeter/5.3/libexec/bin/user.properties +2020-10-16 21:40:47,248 INFO o.a.j.JMeter: Loading system properties from: /usr/local/Cellar/jmeter/5.3/libexec/bin/system.properties +2020-10-16 21:40:47,249 INFO o.a.j.JMeter: Copyright (c) 1998-2020 The Apache Software Foundation +2020-10-16 21:40:47,249 INFO o.a.j.JMeter: Version 5.3 +2020-10-16 21:40:47,249 INFO o.a.j.JMeter: java.version=11.0.8 +2020-10-16 21:40:47,249 INFO o.a.j.JMeter: java.vm.name=OpenJDK 64-Bit Server VM +2020-10-16 21:40:47,249 INFO o.a.j.JMeter: os.name=Mac OS X +2020-10-16 21:40:47,249 INFO o.a.j.JMeter: os.arch=x86_64 +2020-10-16 21:40:47,249 INFO o.a.j.JMeter: os.version=10.15.3 +2020-10-16 21:40:47,250 INFO o.a.j.JMeter: file.encoding=UTF-8 +2020-10-16 21:40:47,250 INFO o.a.j.JMeter: java.awt.headless=null +2020-10-16 21:40:47,250 INFO o.a.j.JMeter: Max memory =1073741824 +2020-10-16 21:40:47,250 INFO o.a.j.JMeter: Available Processors =8 +2020-10-16 21:40:47,262 INFO o.a.j.JMeter: Default Locale=English (EN) +2020-10-16 21:40:47,262 INFO o.a.j.JMeter: JMeter Locale=English (EN) +2020-10-16 21:40:47,262 INFO o.a.j.JMeter: JMeterHome=/usr/local/Cellar/jmeter/5.3/libexec +2020-10-16 21:40:47,262 INFO o.a.j.JMeter: user.dir =/Users/vishalb/code/fission_codebase/examples/samples/long-running-compute +2020-10-16 21:40:47,263 INFO o.a.j.JMeter: PWD =/Users/vishalb/code/fission_codebase/examples/samples/long-running-compute +2020-10-16 21:40:47,267 INFO o.a.j.JMeter: IP: 127.0.0.1 Name: Vishals-MacBook-Pro.local FullName: localhost +2020-10-16 21:40:47,982 INFO o.a.j.JMeter: Setting LAF to: com.github.weisj.darklaf.DarkLaf:com.github.weisj.darklaf.theme.DarculaTheme +2020-10-16 21:40:50,151 INFO o.a.j.JMeter: Loaded icon properties from org/apache/jmeter/images/icon.properties +2020-10-16 21:40:51,393 INFO o.j.r.JARSourceHTTP: Requesting https://jmeter-plugins.org/repo/?installID=mac_os_x-1007122775c217cc51aec5a93e7c81ca-gui +2020-10-16 21:40:54,058 INFO o.j.r.PluginManager: Plugins Status: [jpgc-plugins-manager=1.3, jmeter-core=5.3, jmeter-ftp=5.3, jmeter-http=5.3, jmeter-jdbc=5.3, jmeter-jms=5.3, jmeter-junit=5.3, jmeter-java=5.3, jmeter-ldap=5.3, jmeter-mail=5.3, jmeter-mongodb=5.3, jmeter-native=5.3, jmeter-tcp=5.3, jmeter-components=5.3] +2020-10-16 21:40:54,059 INFO o.j.r.PluginManagerMenuItem: Plugins Manager has upgrades: [jpgc-plugins-manager] +2020-10-16 21:41:08,331 INFO o.a.j.s.FileServer: Default base='/Users/vishalb/code/fission_codebase/examples/samples/long-running-compute' +2020-10-16 21:41:08,334 INFO o.a.j.g.a.Load: Loading file: /Users/vishalb/Downloads/Fission.jmx +2020-10-16 21:41:08,335 INFO o.a.j.s.FileServer: Set new base='/Users/vishalb/Downloads' +2020-10-16 21:41:08,488 INFO o.a.j.s.SaveService: Testplan (JMX) version: 2.2. Testlog (JTL) version: 2.2 +2020-10-16 21:41:08,499 INFO o.a.j.s.SaveService: Using SaveService properties version 5.0 +2020-10-16 21:41:08,501 INFO o.a.j.s.SaveService: Using SaveService properties file encoding UTF-8 +2020-10-16 21:41:08,502 INFO o.a.j.s.SaveService: Loading file: /Users/vishalb/Downloads/Fission.jmx +2020-10-16 21:41:08,905 INFO o.a.j.p.h.s.HTTPSamplerBase: Parser for text/html is org.apache.jmeter.protocol.http.parser.LagartoBasedHtmlParser +2020-10-16 21:41:08,906 INFO o.a.j.p.h.s.HTTPSamplerBase: Parser for application/xhtml+xml is org.apache.jmeter.protocol.http.parser.LagartoBasedHtmlParser +2020-10-16 21:41:08,906 INFO o.a.j.p.h.s.HTTPSamplerBase: Parser for application/xml is org.apache.jmeter.protocol.http.parser.LagartoBasedHtmlParser +2020-10-16 21:41:08,906 INFO o.a.j.p.h.s.HTTPSamplerBase: Parser for text/xml is org.apache.jmeter.protocol.http.parser.LagartoBasedHtmlParser +2020-10-16 21:41:08,906 INFO o.a.j.p.h.s.HTTPSamplerBase: Parser for text/vnd.wap.wml is org.apache.jmeter.protocol.http.parser.RegexpHTMLParser +2020-10-16 21:41:08,906 INFO o.a.j.p.h.s.HTTPSamplerBase: Parser for text/css is org.apache.jmeter.protocol.http.parser.CssParser +2020-10-16 21:41:09,065 INFO o.a.j.s.SampleResult: Note: Sample TimeStamps are START times +2020-10-16 21:41:09,066 INFO o.a.j.s.SampleResult: sampleresult.default.encoding is set to ISO-8859-1 +2020-10-16 21:41:09,066 INFO o.a.j.s.SampleResult: sampleresult.useNanoTime=true +2020-10-16 21:41:09,066 INFO o.a.j.s.SampleResult: sampleresult.nanoThreadSleep=5000 +2020-10-16 21:41:09,224 INFO o.a.j.v.ViewResultsFullVisualizer: Add JavaFX to your Java installation if you want to use renderer: org.apache.jmeter.visualizers.RenderInBrowser +2020-10-16 21:41:09,295 INFO o.a.j.s.FileServer: Set new base='/Users/vishalb/Downloads' +2020-10-16 21:42:01,551 INFO o.a.j.e.StandardJMeterEngine: Running the test! +2020-10-16 21:42:01,552 INFO o.a.j.s.SampleEvent: List of sample_variables: [] +2020-10-16 21:42:01,552 INFO o.a.j.s.SampleEvent: List of sample_variables: [] +2020-10-16 21:42:01,557 INFO o.a.j.e.u.CompoundVariable: Note: Function class names must contain the string: '.functions.' +2020-10-16 21:42:01,557 INFO o.a.j.e.u.CompoundVariable: Note: Function class names must not contain the string: '.gui.' +2020-10-16 21:42:01,861 INFO o.a.j.g.u.JMeterMenuBar: setRunning(true, *local*) +2020-10-16 21:42:01,914 INFO o.a.j.e.StandardJMeterEngine: Starting ThreadGroup: 1 : Function#1 +2020-10-16 21:42:01,914 INFO o.a.j.e.StandardJMeterEngine: Starting 1 threads for group Function#1. +2020-10-16 21:42:01,914 INFO o.a.j.e.StandardJMeterEngine: Thread will continue on error +2020-10-16 21:42:01,915 INFO o.a.j.t.ThreadGroup: Starting thread group... number=1 threads=1 ramp-up=1 delayedStart=false +2020-10-16 21:42:01,919 INFO o.a.j.t.ThreadGroup: Started thread group number 1 +2020-10-16 21:42:01,919 INFO o.a.j.e.StandardJMeterEngine: All thread groups have been started +2020-10-16 21:42:01,920 INFO o.a.j.t.JMeterThread: Thread started: Function#1 1-1 +2020-10-16 21:42:01,931 INFO o.a.j.p.h.s.HTTPHCAbstractImpl: Local host = Vishals-MacBook-Pro.local +2020-10-16 21:42:01,938 INFO o.a.j.p.h.s.HTTPHC4Impl: HTTP request retry count = 0 +2020-10-16 21:42:18,357 INFO o.a.j.t.JMeterThread: Thread is done: Function#1 1-1 +2020-10-16 21:42:18,357 INFO o.a.j.t.JMeterThread: Thread finished: Function#1 1-1 +2020-10-16 21:42:18,358 INFO o.a.j.e.StandardJMeterEngine: Notifying test listeners of end of test +2020-10-16 21:42:18,359 INFO o.a.j.g.u.JMeterMenuBar: setRunning(false, *local*) diff --git a/samples/long-running-compute/specs/env-python-1.yaml b/samples/long-running-compute/specs/env-python-1.yaml index 72dbdae..8ac792f 100644 --- a/samples/long-running-compute/specs/env-python-1.yaml +++ b/samples/long-running-compute/specs/env-python-1.yaml @@ -8,7 +8,7 @@ spec: builder: {} imagepullsecret: "" keeparchive: false - poolsize: 250 + poolsize: 25 resources: limits: cpu: 75m @@ -17,6 +17,6 @@ spec: cpu: 50m memory: 64Mi runtime: - image: fission/python-env + image: fission/python-env:1.11.0 terminationGracePeriod: 1 version: 3 \ No newline at end of file diff --git a/samples/long-running-compute/specs/env-python-2.yaml b/samples/long-running-compute/specs/env-python-2.yaml index f300496..81b6033 100644 --- a/samples/long-running-compute/specs/env-python-2.yaml +++ b/samples/long-running-compute/specs/env-python-2.yaml @@ -8,7 +8,7 @@ spec: builder: {} imagepullsecret: "" keeparchive: false - poolsize: 150 + poolsize: 15 resources: limits: cpu: 100m @@ -17,6 +17,6 @@ spec: cpu: 75m memory: 96Mi runtime: - image: fission/python-env + image: fission/python-env:1.11.0 terminationGracePeriod: 1 version: 3 \ No newline at end of file diff --git a/samples/long-running-compute/specs/env-python-3.yaml b/samples/long-running-compute/specs/env-python-3.yaml index 5fb5816..dea223e 100644 --- a/samples/long-running-compute/specs/env-python-3.yaml +++ b/samples/long-running-compute/specs/env-python-3.yaml @@ -8,7 +8,7 @@ spec: builder: {} imagepullsecret: "" keeparchive: false - poolsize: 150 + poolsize: 15 resources: limits: cpu: 75m @@ -17,6 +17,6 @@ spec: cpu: 50m memory: 64Mi runtime: - image: fission/python-env + image: fission/python-env:1.11.0 terminationGracePeriod: 1 version: 3 \ No newline at end of file diff --git a/samples/long-running-compute/specs/env-python-4.yaml b/samples/long-running-compute/specs/env-python-4.yaml index 6e95b57..60e6918 100644 --- a/samples/long-running-compute/specs/env-python-4.yaml +++ b/samples/long-running-compute/specs/env-python-4.yaml @@ -8,7 +8,7 @@ spec: builder: {} imagepullsecret: "" keeparchive: false - poolsize: 100 + poolsize: 10 resources: limits: cpu: 75m @@ -17,6 +17,6 @@ spec: cpu: 50m memory: 64Mi runtime: - image: fission/python-env + image: fission/python-env:1.11.0 terminationGracePeriod: 1 version: 3 \ No newline at end of file diff --git a/samples/long-running-compute/specs/env-python-5.yaml b/samples/long-running-compute/specs/env-python-5.yaml index aa3a01b..f550c62 100644 --- a/samples/long-running-compute/specs/env-python-5.yaml +++ b/samples/long-running-compute/specs/env-python-5.yaml @@ -8,7 +8,7 @@ spec: builder: {} imagepullsecret: "" keeparchive: false - poolsize: 100 + poolsize: 10 resources: limits: cpu: 75m @@ -17,6 +17,6 @@ spec: cpu: 50m memory: 64Mi runtime: - image: fission/python-env + image: fission/python-env:1.11.0 terminationGracePeriod: 1 version: 3 \ No newline at end of file diff --git a/samples/long-running-compute/specs/function-hello-1-15.yaml b/samples/long-running-compute/specs/function-hello-1-15.yaml index 1854ad6..ebef3aa 100644 --- a/samples/long-running-compute/specs/function-hello-1-15.yaml +++ b/samples/long-running-compute/specs/function-hello-1-15.yaml @@ -40,7 +40,7 @@ spec: SpecializationTimeout: 3600 TargetCPUPercent: 0 StrategyType: execution - concurrency: 2000 + concurrency: 200 configmaps: null environment: name: python-1 diff --git a/samples/long-running-compute/specs/function-hello-2-45.yaml b/samples/long-running-compute/specs/function-hello-2-45.yaml index af0bafb..6ac7e86 100644 --- a/samples/long-running-compute/specs/function-hello-2-45.yaml +++ b/samples/long-running-compute/specs/function-hello-2-45.yaml @@ -40,7 +40,7 @@ spec: SpecializationTimeout: 3600 TargetCPUPercent: 0 StrategyType: execution - concurrency: 1200 + concurrency: 120 configmaps: null environment: name: python-2 diff --git a/samples/long-running-compute/specs/function-hello-3-75.yaml b/samples/long-running-compute/specs/function-hello-3-75.yaml index a9bd4a3..34df712 100644 --- a/samples/long-running-compute/specs/function-hello-3-75.yaml +++ b/samples/long-running-compute/specs/function-hello-3-75.yaml @@ -40,7 +40,7 @@ spec: SpecializationTimeout: 3600 TargetCPUPercent: 0 StrategyType: execution - concurrency: 1200 + concurrency: 120 configmaps: null environment: name: python-3 diff --git a/samples/long-running-compute/specs/function-hello-4-90.yaml b/samples/long-running-compute/specs/function-hello-4-90.yaml index b8dc995..3380cfc 100644 --- a/samples/long-running-compute/specs/function-hello-4-90.yaml +++ b/samples/long-running-compute/specs/function-hello-4-90.yaml @@ -40,7 +40,7 @@ spec: SpecializationTimeout: 3600 TargetCPUPercent: 0 StrategyType: execution - concurrency: 800 + concurrency: 80 configmaps: null environment: name: python-4 diff --git a/samples/long-running-compute/specs/function-hello-5-120.yaml b/samples/long-running-compute/specs/function-hello-5-120.yaml index 96bbe86..c98c4cf 100644 --- a/samples/long-running-compute/specs/function-hello-5-120.yaml +++ b/samples/long-running-compute/specs/function-hello-5-120.yaml @@ -40,7 +40,7 @@ spec: SpecializationTimeout: 3600 TargetCPUPercent: 0 StrategyType: execution - concurrency: 800 + concurrency: 80 configmaps: null environment: name: python-5 diff --git a/scripts/cluster-setup/README.md b/scripts/cluster-setup/README.md index 9577c8c..02781d5 100644 --- a/scripts/cluster-setup/README.md +++ b/scripts/cluster-setup/README.md @@ -1,61 +1,9 @@ -echo "GET http://a72c0989896a24498b58842b22da42a6-2126244409.ap-south-1.elb.amazonaws.com/fission-function/hello-1-15" | vegeta attack -duration=300s -timeout=3600s -rate=2500/m -max-workers=2000> hello-1-15.txt & +echo "GET http://LB_ADDRESS/fission-function/hello-1-15" | vegeta attack -duration=300s -timeout=3600s -rate=2500/1m -max-workers=13000> hello-1-15.txt & -echo "GET http://a72c0989896a24498b58842b22da42a6-2126244409.ap-south-1.elb.amazonaws.com/fission-function/hello-2-45" | vegeta attack -duration=300s -timeout=3600s -rate=1500/m -max-workers=2000> hello-2-45.txt & +echo "GET http://LB_ADDRESS/fission-function/hello-2-45" | vegeta attack -duration=300s -timeout=3600s -rate=1500/1m -max-workers=8000> hello-2-45.txt & -echo "GET http://a72c0989896a24498b58842b22da42a6-2126244409.ap-south-1.elb.amazonaws.com/fission-function/hello-3-75" | vegeta attack -duration=300s -timeout=3600s -rate=1500/m -max-workers=2000> hello-3-75.txt & +echo "GET http://LB_ADDRESS/fission-function/hello-3-75" | vegeta attack -duration=300s -timeout=3600s -rate=1500/1m -max-workers=8000> hello-3-75.txt & -echo "GET http://a72c0989896a24498b58842b22da42a6-2126244409.ap-south-1.elb.amazonaws.com/fission-function/hello-4-90" | vegeta attack -duration=300s -timeout=4800s -rate=1000/m -max-workers=2000> hello-4-90.txt & +echo "GET http://LB_ADDRESS/fission-function/hello-4-90" | vegeta attack -duration=300s -timeout=4800s -rate=1000/1m -max-workers=6000> hello-4-90.txt & -echo "GET http://a72c0989896a24498b58842b22da42a6-2126244409.ap-south-1.elb.amazonaws.com/fission-function/hello-5-120" | vegeta attack -duration=300s -timeout=4800s -rate=1000/m -max-workers=2000> hello-5-120.txt & - -root@test-temp:~# vegeta report hello-5-120.txt -Requests [total, rate, throughput] 2001, 2.03, 0.59 -Duration [total, attack, wait] 49m42s, 16m27s, 33m15s -Latencies [min, mean, 50, 90, 95, 99, max] 16m26s, 36m24s, 38m58s, 45m26s, 46m29s, 47m3s, 47m9s -Bytes In [total, mean] 20325, 10.16 -Bytes Out [total, mean] 0, 0.00 -Success [ratio] 87.86% -Status Codes [code:count] 200:1758 502:243 -Error Set: -502 Bad Gateway - -root@test-temp:~# vegeta report hello-4-90.txt -Requests [total, rate, throughput] 2001, 2.11, 0.65 -Duration [total, attack, wait] 49m15s, 15m49s, 33m26s -Latencies [min, mean, 50, 90, 95, 99, max] 15m47s, 35m16s, 37m54s, 45m12s, 45m53s, 46m27s, 46m35s -Bytes In [total, mean] 15827, 7.91 -Bytes Out [total, mean] 0, 0.00 -Success [ratio] 96.50% -Status Codes [code:count] 200:1931 502:70 -Error Set: -502 Bad Gateway - -root@test-temp:~# vegeta report hello-3-75.txt -Requests [total, rate, throughput] 2001, 2.38, 0.68 -Duration [total, attack, wait] 49m2s, 13m59s, 35m3s -Latencies [min, mean, 50, 90, 95, 99, max] 13m59s, 29m45s, 31m12s, 40m9s, 41m7s, 41m52s, 41m58s -Bytes In [total, mean] 14007, 7.00 -Bytes Out [total, mean] 0, 0.00 -Success [ratio] 100.00% -Status Codes [code:count] 200:2001 -Error Set: - -root@test-temp:~# vegeta report hello-2-45.txt -Requests [total, rate, throughput] 2001, 2.47, 0.69 -Duration [total, attack, wait] 48m34s, 13m29s, 35m5s -Latencies [min, mean, 50, 90, 95, 99, max] 13m29s, 28m13s, 29m51s, 38m44s, 39m53s, 40m55s, 41m30s -Bytes In [total, mean] 14007, 7.00 -Bytes Out [total, mean] 0, 0.00 -Success [ratio] 100.00% -Status Codes [code:count] 200:2001 -Error Set: - -root@test-temp:~# vegeta report hello-1-15.txt -Requests [total, rate, throughput] 2001, 2.68, 0.68 -Duration [total, attack, wait] 48m59s, 12m26s, 36m34s -Latencies [min, mean, 50, 90, 95, 99, max] 12m26s, 21m12s, 20m50s, 28m59s, 29m58s, 39m4s, 48m16s -Bytes In [total, mean] 14007, 7.00 -Bytes Out [total, mean] 0, 0.00 -Success [ratio] 100.00% -Status Codes [code:count] 200:2001 -Error Set: \ No newline at end of file +echo "GET http://LB_ADDRESS/fission-function/hello-5-120" | vegeta attack -duration=300s -timeout=4800s -rate=1000/1m -max-workers=6000> hello-5-120.txt & diff --git a/scripts/cluster-setup/cluster-as.yaml b/scripts/cluster-setup/cluster-as.yaml index b30b621..e217b9e 100644 --- a/scripts/cluster-setup/cluster-as.yaml +++ b/scripts/cluster-setup/cluster-as.yaml @@ -155,7 +155,7 @@ spec: - --cloud-provider=aws - --skip-nodes-with-local-storage=false - --expander=least-waste - - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/scale-test + - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/fission-scale - --balance-similar-node-groups - --skip-nodes-with-system-pods=false volumeMounts: diff --git a/scripts/cluster-setup/setupcluster.sh b/scripts/cluster-setup/setupcluster.sh index 005a3dd..ebe4c2f 100755 --- a/scripts/cluster-setup/setupcluster.sh +++ b/scripts/cluster-setup/setupcluster.sh @@ -4,7 +4,7 @@ set -euo pipefail echo "Starting Eksctl cluster setup" # eksctl create cluster -f cluster.yaml # Till --asg-access is supported in YAML file, we need to use CLI -eksctl create cluster --name scale-test --version 1.17 --managed --asg-access --region ap-south-1 --node-type=c4.4xlarge --nodes-min 3 --nodes-max 20 --ssh-access +eksctl create cluster --name fission-scale --version 1.17 --managed --asg-access --region ap-south-1 --node-type=c4.4xlarge --nodes-min 1 --nodes-max 20 --ssh-access echo "Getting nodes" kubectl get nodes @@ -20,7 +20,9 @@ helm install fission-metrics --namespace monitoring prometheus-community/kube-pr --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \ --set prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues=false -# helm install --namespace fission --name-template fission --set prometheus.enabled=false https://github.com/fission/fission/releases/download/1.10.0/fission-all-1.10.0.tgz -# kubectl annotate svc/router -nfission service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "3600" +helm install --namespace fission --name-template fission --set prometheus.enabled=false,nats.enabled=false,executor.podReadyTimeout=3600s,router.unTapServiceTimeout=4800s https://github.com/fission/fission/releases/download/1.11.2/fission-all-1.11.2.tgz + +# kubectl annotate svc/router -nfission service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "4800" + # kubectl apply -f servicemonitors.yaml --namespace fission # kubectl --namespace monitoring port-forward svc/fission-metrics-grafana 3000:80 \ No newline at end of file diff --git a/scripts/cluster-setup/setuptestcluster.sh b/scripts/cluster-setup/setuptestcluster.sh deleted file mode 100644 index a077c53..0000000 --- a/scripts/cluster-setup/setuptestcluster.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -set -euo pipefail diff --git a/scripts/cluster-setup/sys_changes_ds.yaml b/scripts/cluster-setup/sys_changes_ds.yaml new file mode 100644 index 0000000..f730380 --- /dev/null +++ b/scripts/cluster-setup/sys_changes_ds.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + name: syschanges + name: system-changes-sysds-node-exporter +spec: + selector: + matchLabels: + component: "node-exporter" + app: sysds + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + component: "node-exporter" + app: sysds + heritage: Helm + spec: + containers: + - name: busybox + image: "busybox" + command: + - /bin/sh + - -c + - | + sysctl -w net.ipv4.tcp_max_tw_buckets=65535 + #sysctl -w net.ipv4.tcp_tw_recycle=1 + sysctl -w net.ipv4.tcp_tw_reuse=0 + sysctl -w net.ipv4.tcp_max_syn_backlog=131072 + sysctl -w net.ipv4.tcp_syn_retries=3 + sysctl -w net.ipv4.tcp_synack_retries=3 + sysctl -w net.ipv4.tcp_retries1=3 + sysctl -w net.ipv4.tcp_retries2=8 + sysctl -w net.ipv4.tcp_rmem="16384 174760 349520" + sysctl -w net.ipv4.tcp_wmem="16384 131072 262144" + sysctl -w net.ipv4.tcp_mem="262144 524288 1048576" + sysctl -w net.ipv4.tcp_max_orphans=65536 + sysctl -w net.ipv4.tcp_fin_timeout=10 + sysctl -w net.ipv4.tcp_low_latency=1 + sysctl -w net.ipv4.tcp_syncookies=0 + imagePullPolicy: "IfNotPresent" + securityContext: + privileged: true + resources: + {} + volumeMounts: + - name: proc + mountPath: /host/proc + readOnly: true + - name: sys + mountPath: /host/sys + readOnly: true + hostNetwork: true + hostPID: true + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys \ No newline at end of file diff --git a/scripts/cluster-setup/test/README.md b/scripts/cluster-setup/test/README.md new file mode 100644 index 0000000..b76138f --- /dev/null +++ b/scripts/cluster-setup/test/README.md @@ -0,0 +1,41 @@ +# Running Tests + +1) OS Preps & Tuning + +``` +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61 +echo "deb https://dl.bintray.com/loadimpact/deb stable main" | sudo tee -a /etc/apt/sources.list +sudo apt-get update +sudo apt-get install k6 +``` + +``` +sysctl -w net.ipv4.ip_local_port_range="1024 65535" +sysctl -w net.ipv4.tcp_tw_reuse=1 +sysctl -w net.ipv4.tcp_timestamps=1 +sysctl -w net.core.rmem_max = 16777216 +sysctl -w net.core.wmem_max = 16777216 +sysctl -w net.ipv4.tcp_rmem = 4096 87380 16777216 +sysctl -w net.ipv4.tcp_wmem = 4096 65536 16777216 +sysctl -w net.ipv4.tcp_syncookies = 1 +sysctl -w net.ipv4.tcp_mem = "50576 64768 98152" +sysctl -w net.core.netdev_max_backlog = 2500 +sysctl -w net.ipv4.netfilter.ip_conntrack_max = 1048576 + +sysctl -p + +ulimit -n 250000 + +echo "1024 65535" > /proc/sys/net/ipv4/iplocalport_range + +``` + +``` +apt-get install -y htop iftop +``` + +) Running test + +k6 run -e MY_HOSTNAME=LB_ADDRESS --summary-export export.json test-script.js + +) Analyzing Results \ No newline at end of file diff --git a/scripts/cluster-setup/test/export.json b/scripts/cluster-setup/test/export.json new file mode 100644 index 0000000..302aead --- /dev/null +++ b/scripts/cluster-setup/test/export.json @@ -0,0 +1,114 @@ +{ + "metrics": { + "checks": { + "fails": 4927, + "passes": 78, + "value": 0 + }, + "data_received": { + "count": 10998, + "rate": 30.51122133947164 + }, + "data_sent": { + "count": 917921, + "rate": 2546.5439901026684 + }, + "http_req_blocked": { + "avg": 1300.5288843156866, + "max": 21112.443, + "med": 13.904, + "min": 0, + "p(90)": 2216.3966000000005, + "p(95)": 12167.5162 + }, + "http_req_connecting": { + "avg": 1299.4451564435537, + "max": 21100.119, + "med": 13.186, + "min": 0, + "p(90)": 2215.6422000000002, + "p(95)": 12166.099199999999 + }, + "http_req_duration": { + "avg": 59076.74311828164, + "max": 60223.502, + "med": 59560.425, + "min": 0, + "p(90)": 59959.4418, + "p(95)": 59971.7874 + }, + "http_req_receiving": { + "avg": 0.004345454545454547, + "max": 1.111, + "med": 0, + "min": 0, + "p(90)": 0, + "p(95)": 0 + }, + "http_req_sending": { + "avg": 0.8668423576423586, + "max": 46.064, + "med": 0.183, + "min": 0, + "p(90)": 1.7814000000000014, + "p(95)": 3.069800000000001 + }, + "http_req_tls_handshaking": { + "avg": 0, + "max": 0, + "med": 0, + "min": 0, + "p(90)": 0, + "p(95)": 0 + }, + "http_req_waiting": { + "avg": 59075.87193046936, + "max": 60223.415, + "med": 59559.2, + "min": 0, + "p(90)": 59958.7206, + "p(95)": 59971.1964 + }, + "http_reqs": { + "count": 5005, + "rate": 13.885130278601158 + }, + "iteration_duration": { + "avg": 60508.28998786842, + "max": 81174.807309, + "med": 59759.579119, + "min": 22727.599003, + "p(90)": 61585.9261736, + "p(95)": 71465.489313 + }, + "iterations": { + "count": 5005, + "rate": 13.885130278601158 + }, + "vus": { + "max": 3750, + "min": 2750, + "value": 2750 + }, + "vus_max": { + "max": 3750, + "min": 3750, + "value": 3750 + } + }, + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": {}, + "checks": { + "status is 200": { + "name": "status is 200", + "path": "::status is 200", + "id": "6210a8cd14cd70477eba5c5e4cb3fb5f", + "passes": 78, + "fails": 4927 + } + } + } +} diff --git a/scripts/cluster-setup/test/targets.conf b/scripts/cluster-setup/test/targets.conf new file mode 100644 index 0000000..42c59a8 --- /dev/null +++ b/scripts/cluster-setup/test/targets.conf @@ -0,0 +1,6 @@ +# vegeta attack -duration=300s -timeout=3600s -rate=1000/1m -max-workers=5000 -targets=targets.conf > vtest.txt +GET http://LB_ADDRESS/fission-function/hello-1-15 +GET http://LB_ADDRESS/fission-function/hello-2-45 +GET http://LB_ADDRESS/fission-function/hello-3-75 +GET http://LB_ADDRESS/fission-function/hello-4-90 +GET http://LB_ADDRESS/fission-function/hello-5-120 \ No newline at end of file diff --git a/scripts/cluster-setup/test/test-script.js b/scripts/cluster-setup/test/test-script.js new file mode 100644 index 0000000..7a7dd79 --- /dev/null +++ b/scripts/cluster-setup/test/test-script.js @@ -0,0 +1,98 @@ +import http from "k6/http"; +import { check } from "k6"; + +export let options = { + noConnectionReuse: true, + noVUConnectionReuse: true, + summaryTrendStats: [`avg`, `min`, `med`, `max`, `p(5)`, `p(10)`, `p(15)`, `p(20)`, `p(25)`, `p(30)`], + linger: true, + teardownTimeout: '90m', + scenarios: { + func1: { + executor: 'constant-arrival-rate', + exec: 'func1_exec', + rate: 500, + preAllocatedVUs: 1250, + timeUnit: '1m', + duration: '5m', + gracefulStop: '90m', + }, + func2: { + executor: 'constant-arrival-rate', + exec: 'func2_exec', + rate: 150, + preAllocatedVUs: 750, + timeUnit: '1m', + duration: '5m', + gracefulStop: '90m', + }, + func3: { + executor: 'constant-arrival-rate', + exec: 'func3_exec', + rate: 150, + preAllocatedVUs: 750, + timeUnit: '1m', + duration: '5m', + gracefulStop: '90m', + }, + func4: { + executor: 'constant-arrival-rate', + exec: 'func4_exec', + rate: 100, + preAllocatedVUs: 500, + timeUnit: '1m', + duration: '5m', + gracefulStop: '90m', + }, + func5: { + executor: 'constant-arrival-rate', + exec: 'func5_exec', + rate: 100, + preAllocatedVUs: 500, + timeUnit: '1m', + duration: '5m', + gracefulStop: '90m', + } + }, +}; + + +export function func1_exec() { + let params = { timeout: 3600000 } + let res = http.get(`http://${__ENV.MY_HOSTNAME}/fission-function/hello-1-15`, params) + check(res, { + "status is 200": (r) => r.status === 200 + }); +} + +export function func2_exec() { + let params = { timeout: 3600000 } + let res = http.get(`http://${__ENV.MY_HOSTNAME}/fission-function/hello-2-45`, params) + check(res, { + "status is 200": (r) => r.status === 200 + }); +} + +export function func3_exec() { + let params = { timeout: 3600000 } + let res = http.get(`http://${__ENV.MY_HOSTNAME}/fission-function/hello-3-75`, params) + check(res, { + "status is 200": (r) => r.status === 200 + }); +} + +export function func4_exec() { + let params = { timeout: 3600000 } + let res = http.get(`http://${__ENV.MY_HOSTNAME}/fission-function/hello-4-90`, params) + check(res, { + "status is 200": (r) => r.status === 200 + }); +} + +export function func5_exec() { + let params = { timeout: 3600000 } + let res = http.get(`http://${__ENV.MY_HOSTNAME}/fission-function/hello-5-120`, params) + check(res, { + "status is 200": (r) => r.status === 200 + }); +} \ No newline at end of file diff --git a/scripts/cluster-setup/test/vtest.txt b/scripts/cluster-setup/test/vtest.txt new file mode 100644 index 0000000..d634f7c Binary files /dev/null and b/scripts/cluster-setup/test/vtest.txt differ diff --git a/scripts/cluster-setup/test/vtest_10pm.txt b/scripts/cluster-setup/test/vtest_10pm.txt new file mode 100644 index 0000000..434bb2c Binary files /dev/null and b/scripts/cluster-setup/test/vtest_10pm.txt differ diff --git a/scripts/cluster-setup/test/vtest_7pm.txt b/scripts/cluster-setup/test/vtest_7pm.txt new file mode 100644 index 0000000..faa2221 Binary files /dev/null and b/scripts/cluster-setup/test/vtest_7pm.txt differ diff --git a/scripts/cluster-setup/test/vtest_8pm.txt b/scripts/cluster-setup/test/vtest_8pm.txt new file mode 100644 index 0000000..b2116df Binary files /dev/null and b/scripts/cluster-setup/test/vtest_8pm.txt differ