-
Notifications
You must be signed in to change notification settings - Fork 24
/
Copy pathclusterbuster
executable file
·5001 lines (4721 loc) · 158 KB
/
clusterbuster
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
# Copyright 2019-2022 Robert Krawitz/Red Hat
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Find our helpers
function finddir() {
local path_to_file
path_to_file=$(readlink -f "$0")
if [[ -z $path_to_file ]] ; then
return 1
elif [[ -d $path_to_file ]] ; then
echo "$path_to_file/"
elif [[ -e $path_to_file ]] ; then
echo "${path_to_file%/*}/"
else
return 1
fi
return 0
}
declare __realsc__=
declare __topsc__
if [[ -z ${__topsc__:-} ]] ; then
export __topsc__="${0##*/}"
# shellcheck disable=SC2155
export __topdir__="$(finddir "$0")"
[[ -z $__topdir__ ]] && fatal "Can't find directory for $0"
fi
function clean_startup() {
[[ -f $__realsc__ ]] && rm -f "$__realsc__"
}
# This allows us to edit the script while another instance is running
# since this script sticks around until the user exits the spawned shell.
# It's fine for the running script to be removed, since the shell still
# has its open file descriptor.
if [[ $# = 0 || $1 != "--DoIt=$0" ]] ; then
tmpsc=$(mktemp -t "${__topsc__}".XXXXXXXXXX)
[[ -z $tmpsc || ! -f $tmpsc || -L $tmpsc ]] && fatal "Can't create temporary script file"
trap clean_startup EXIT SIGHUP SIGINT SIGQUIT SIGTERM
PATH+=${PATH:+:}$__topdir__
cat "$0" > "$tmpsc"
chmod +x "$tmpsc"
exec "$tmpsc" "--DoIt=$tmpsc" "$@"
else
# Needed for stack traceback to get function arguments
shopt -s extdebug
__realsc__=${1#--DoIt=}
clean_startup
export -n __topsc__ __topdir__
shift
fi
# Unfortunate that there's no way to tell shellcheck to always source
# specific files.
# shellcheck disable=SC2034
# See the bash change log, differences between 4.4-beta2 and 4.4-rc2:
# a. Using ${a[@]} or ${a[*]} with an array without any assigned elements when
# the nounset option is enabled no longer throws an unbound variable error.
if (( BASH_VERSINFO[0] >= 5 || (BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] >= 4) )) ; then
set -u
else
cat 1>&2 <<EOF
Warning: bash version at least 4.4 is recommended for using ${__topsc__##*/}.
Actual version is ${BASH_VERSION}
EOF
fi
declare ___arg
for ___arg in "$@" ; do
if [[ "${___arg:-}" = '--force-abort'* ]] ; then
echo "*** WARNING: will abort on any shell error!" 1>&2
set -e
set -o errtrace
#trap 'killthemall "Caught error, aborting!"' ERR
break
fi
done
declare -i namespaces=1
declare -i use_namespaces=1
declare -i deps_per_namespace=1
declare -i remove_namespaces=-1
declare -i create_namespaces_only=0
declare -i secrets=0
declare -i replicas=1
declare -i parallel=1
declare -i first_deployment=0
declare -i sleep_between_secrets=0
declare -i sleep_between_configmaps=0
declare -i sleep_between_namespaces=0
declare -i sleep_between_deployments=0
declare -i parallel_secrets=0
declare -i parallel_configmaps=0
declare -i parallel_namespaces=0
declare -i parallel_deployments=0
declare -i parallel_log_retrieval=50
declare -i retrieve_successful_logs=0
declare -i objs_per_call=1
declare -i objs_per_call_secrets=0
declare -i objs_per_call_configmaps=0
declare -i objs_per_call_namespaces=0
declare -i objs_per_call_deployments=0
declare -i containers_per_pod=1
declare -i sleeptime=0
declare -i doit=1
declare -i objs_item_count=0
declare -i port=7777
declare -i sync_port=7778
declare -i drop_cache_port=7779
declare -i sync_watchdog_port_num=7780
declare -i sync_watchdog_timeout=0
declare -i sync_ns_port=7753
declare -i sync_in_first_namespace=0
declare sync_host=
declare -i affinity=0
declare -i sync_affinity=0
declare -A pin_nodes=()
declare -A runtime_classes=()
declare -A net_interfaces=()
declare -r default_net_interface_pod=net1
declare -r default_net_interface_vm=eth1
declare runtime_class=
declare scheduler=
declare -i verbose=0
declare -i wait_for_secrets=1
declare -i bytes_transfer=0
declare -i bytes_transfer_max=0
# shellcheck disable=SC2034
declare -i default_bytes_transfer=1000000000
declare -i workload_run_time=0
declare -i workload_run_time_max=0
declare -i exit_at_end=1
declare -i report_object_creation=1
declare -A objects_created=()
declare baseoffset=0
declare -i metrics_epoch=0
declare requested_workload=
declare basename=${CLUSTERBUSTER_DEFAULT_BASENAME:-clusterbuster}
declare deployment_type=pod
declare basetime
declare opt
declare -r nl=$'\n'
declare -a resource_requests=()
declare -a resource_limits=()
declare -A namespaces_in_use=()
declare -a namespaces_to_create=()
declare sync_namespace=
declare -i scale_ns=0
declare -i scale_deployments=1
declare -i sync_start=1
declare -a volumes=()
declare -A mount_volume_map=()
declare common_workdir=/var/opt/clusterbuster
declare -i report=0
declare report_format=summary
declare -i precleanup=1
declare -i cleanup=0
declare -i cleanup_always=0
declare -i timeout=0
declare pathdir=${__topdir__%/*}
declare -a unknown_opts=()
declare -a unknown_opt_names=()
declare -i total_objects_created=0
# <name, filename with contents>
declare -a configmap_files=()
declare -a tolerations=()
declare user_configmap_mount_dir=/etc/clusterbuster
declare system_configmap_mount_dir=/var/lib/clusterbuster
declare -i has_system_configmap=1
declare -i has_user_configmap=1
declare node_selector='node-role.kubernetes.io/worker'
declare -i processes_per_pod=1
declare -i take_prometheus_snapshot=0
declare first_start_timestamp=
declare prometheus_starting_timestamp=
declare prometheus_exact_starting_timestamp=
declare second_start_timestamp=
declare prometheus_ending_timestamp=
declare -i target_data_rate=0
declare job_name=
declare global_sync_service=
declare -i predelay=0
declare -i postdelay=0
declare -r default_metrics_file="metrics-default.yaml"
declare metrics_file=default
declare -i drop_node_cache=0
declare -i drop_all_node_cache=0
declare -i headless_services=1
declare -i virtiofsd_writeback=0
declare -i virtiofsd_direct=1
declare -i virtiofsd_threadpoolsize=0
declare -a virtiofsd_args=()
declare -i liveness_probe_frequency=0
declare -i liveness_probe_sleep_time=0
declare -i metrics_support=-1
declare -i default_pod_start_timeout=60
declare -i default_vm_start_timeout=180
declare -i pod_start_timeout=-1
declare pod_prefix=
declare arch=
declare failure_status=Fail
declare -i create_pods_privileged=0
declare -i wait_forever=0
declare -a pod_labels=()
declare -i get_sync_logs_pid=0
declare -a extra_args=()
declare -i metrics_interval=30
declare -a sync_pod=()
declare workload_step_interval=0
declare -i preserve_tmpdir=0
declare -r ssh_alg=ed25519
declare -A __sysctls=()
declare -i __sysctls_retrieved=0
declare -r sync_flag_file="/tmp/syncfile";
declare -r sync_error_file="/tmp/syncerror";
declare -r controller_timestamp_file="/tmp/timing.json";
declare uuid
uuid=$(uuidgen -r)
declare xuuid=$uuid
# Ensure that pods from other runs don't inadvertently attempt to
# communicate with our sync pod
declare sync_nonce=
sync_nonce=$(uuidgen -r)
declare kata_runtime_class=kata
declare image_pull_policy=
declare container_image='quay.io/rkrawitz/clusterbuster-base:latest'
declare accumulateddata=
declare -a plurals=('s' '')
declare artifactdir=
declare -a saved_argv=("${__topsc__:-0}" "$@")
declare -a processed_options=("${__topsc__:-0}")
declare -a pod_annotations=()
declare -A injected_errors=()
declare cb_tempdir=
declare force_cleanup_timeout=
declare default_namespace_policy=restricted
# vm related variables
declare -i vm_cores=1
declare -i vm_threads=1
declare -i vm_sockets=1
declare -i vm_start_running=1
declare vm_run_strategy=
declare vm_memory=2Gi
declare vm_grace_period=30
declare vm_image='quay.io/rkrawitz/clusterbuster-vm:latest'
declare -i vm_evict_migrate=1
declare -i vm_run_as_container=0
declare -A workload_service_ports=()
declare vm_user=cluster
declare vm_password=buster
declare vm_ssh_keyfile=
declare -i vm_run_as_root=0
# Hard code for the time being
declare -a vm_net_addr=(192 168 129 1)
declare OC=${OC:-${KUBECTL:-}}
OC=${OC:-$(type -p oc)}
OC=${OC:-$(type -p kubectl)} # kubectl might not work, though...
OC=$(type -p "$OC")
declare __virtctl_local_ssh=UNKNOWN
declare VIRTCTL=${VIRTCTL:-$(type -p virtctl)}
function fatal() {
local OPTIND
local waitforit=0
local opt
while getopts 'w' opt "$@" ; do
case "$opt" in
w) waitforit=1 ;;
*) ;;
esac
done
shift "$((OPTIND-1))"
timestamp <<< "$*" 1>&2
if ((waitforit)) ; then
wait
fi
exit 1
}
function warn() {
timestamp <<< "$*" 1>&2
}
if [[ -z "$OC" || ! -x "$OC" ]] ; then
fatal "Can't find kubectl or oc"
fi
declare __libdir__=${__topdir__}/lib/clusterbuster
export CB_LIBPATH=${CB_LIBPATH:-$__libdir__}
[[ -d "$__libdir__" ]] || fatal "Can't find my library dir!"
. "${__libdir__}"/libclusterbuster.sh
function _helpmsg() {
local opt
(IFS=$'\n'; echo "$*"; if [[ -n "$*" ]] ; then echo; fi)
cat <<EOF
Clusterbuster is a tool to permit you to load a configurable workload
onto an OpenShift cluster. ClusterBuster focuses primarily on workload
scalability, including synchronization of multi-instance workloads.
Usage: ${__topsc__:-0} [options] [extra args]
Help:
-h Print basic help information.
-H Print extended help.
Options:
-B basename Base name of pods. Default is
\$CLUSTERBUSTER_DEFAULT_BASENAME if defined or
otherwise 'clusterbuster'.
All objects are labeled with this name.
-E Don't exit after all operations are complete.
-e Exit after all operations are complete (default).
-f jobfile Job file containing settings.
Continuations may be escaped with a trailing
backslash.
A number of examples are provided in the
examples/clusterbuster directory.
-n Print what would be done without doing it
-o Specify report format, as --report-format
-q Do not print verbose log messages (default)
-Q Don't report creation of individual objects (default
report them)
-v Print verbose log messages.
-w workload workload (mandatory), one of:
$(print_workloads ' - ')
--opt[=val] Set the specified option.
Use ${__topsc__##*/} -H to list the available options.
EOF
}
function _help_extended() {
_helpmsg "$@"
cat <<EOF
Extended Options:
General Options (short equivalents):
--doit=<1,0> Run the command or not (default 1) (inverse of -n)
--create-namespaces-only
Only create namespaces.
--jobname=name Name of the job, for logging purposes.
Defaults to the workload name
--workload=type Specify the workload (-P) (mandatory)
--basename=name Specify the base name for any namespaces (-B)
--namespaces=N Number of namespaces
--jobfile=jobfile
Process job file (-f)
--sync Synchronize start of workload instances (default yes)
--precleanup Clean up any prior objects
--cleanup Clean up generated objects unless there's a failure
--cleanup-always Clean up generated objects even if there is a failure
--wait-forever Don't exit if sync pod dies
--remove-namespaces=<1,0>
Remove namespaces when cleaning up objects. Only
applies when using clusterbuster-created namespaces.
By default, namespaces are removed only if no
namespaces previously existed.
--predelay=N Delay for the specified time after workload
starts before it starts running.
--postdelay=N Delay for the specified time after workload
completes.
--stepinterval=N Delay the specified time between steps of the workload.
--timeout=N Time out reporting after N seconds
--report_object_creation=<1,0>
Report creation of individual objects (default 1)
(inverse of -Q)
--uuid=<uuid> Use the specified UUID for the run. Default is to
generate a random-based UUID.
--exit_at_end Exit upon completion of workload (-e/-E)
--verbose Print verbose log messages (-v)
--arch=<architecture>
Use the specified architecture. Default the
architecture of this platform.
--containerimage=<image>
Use the specified container image.
--sync_watchdog_timeout=<timeout>
Set watchdog timer for all pods/VMs. After pods/VMs
start, if the watchdog is not reset within the
timeout, the run is aborted. Currently not set
(timeout = 0); if set to a value greater than
zero, that is the watchdog period.
Reporting Options:
--report=<format>
Print report in specified format. Meaning of
report types is by type. Default is summary
if not specified; if that is not reported,
raw format will be used.
- none
$(list_report_formats ' - ')
The following workloads support reporting:
$(print_workloads_supporting_reporting ' - ')
--artifactdir=<dir>
Save artifacts to <dir>. <dir> can have embedded
format codes:
%n Job name
%s Timestamp of run
%w Workload
%{var} Variable's value is substituted
%{var[item]} to reference an array variable
%{var:-default} to use a default value if not set
--prometheus-snapshot
Take a Prometheus snapshot and save to the
artifacts directory
--metrics[=<file>]
benchmark-runner compatible metrics file
for metrics extraction. If empty or 'none',
no metrics extraction is done. If 'default',
the default
($default_metrics_file)
is used.
--metrics-epoch=<seconds>
Number of seconds to look back for metrics prior
to start of run (default $metrics_epoch)
--metrics-interval=<interval>
Interval between data points for metrics collection.
Default $metrics_interval.
--force-no-metrics
Do not attempt anything that would use metrics
or the prometheus pod.
--failure-status=<status>
Failures should be reported as specified rather
than "Fail"
--retrieve-successful-logs=<0|1>
If retrieving artifacts, retrieve logs for all
pods, not just failing pods. Default $retrieve_successful_logs.
--parallel-logs=n
If retrieving artifacts, parallelize log retrieval.
Workload sizing options:
--containers_per_pod=N
Number of containers per pod
--deployments=N Number of deployments or pods per namespace
--processes=N Number of processes per pod
--replicas=N Number of replicas per deployment
--secrets=N Number of secrets
Generic workload rate options:
--bytestransfer=N[,M]
Number of bytes for workloads operating on
fixed amounts of data.
--targetdatarate=N
Target data rate for workloads operating at fixed
data rates. May have suffixes of K, Ki,
M, Mi, G, Gi, T, or Ti.
--workloadruntime=N
Time to run the workload where applicable
Two comma-separated numbers may be used to
specify maximum time.
Workload placement options:
--pin_node=[class1,class2...]=<node>
Force pod(s) of the specified class(es) onto the
specified node. Multiple comma-separated classes
may be specified. The following classes are
defined for general workloads:
- sync (sync pods)
- client (worker/client pods)
Workloads may define other classes.
If no class is specified, pin node applies to all
pods.
--sync-in-first-namespace=<0|1>
Place the sync pod in the first worker namespace.
Default ${sync_in_first_namespace}.
--affinity Force affinity between client and server pods
in a client-server workload. Default is neither
affinity nor anti-affinity. Use of a pin node
overrides affinity.
--anti-affinity Force anti-affinity between client and server pods
in a client-server workload. Default is neither
affinity nor anti-affinity. Use of a pin node
overrides anti-affinity.
--anti-affinity Force anti-affinity between client and server pods
in a client-server workload. Default is neither
affinity nor anti-affinity. Use of a pin node
overrides anti-affinity.
--sync-affinity Force affinity between sync and all worker pods.
--sync-anti-affinity
Force anti-affinity between sync and all worker pods.
--drop_cache Drop the buffer cache in all pin nodes; if no
pin nodes are defined, drop all workers' caches.
--drop_all_cache Drop the buffer cache on all workers.
Generic workload storage options:
--volume=name:type:mount_path:options
Mount a specified volume.
- name is the name of the volume
- type is the type of volume.
Currently supported volume types are:
- emptydir (pods only; no name required)
- emptydisk (VMs only; no name required)
- pvc or persistentvolumeclaim
- mount_path is the path on which to mount the volume
(required).
Options currently supported include the following:
- claimName is the name of a PVC if it differs
from the volume name. This allows use of
different PVCs; all occurrences of %N
are replaced by the namespace of the pod
mounting the volume; all instances of %i
are replaced by the instance of the pod
within the namespace.
- size in bytes (required for emptydisk; ignored for
other volume types).
- inodes in number (optional for emptydisk; ignored
for other volume types).
- bus (VMs; ignored for pods): bus to be used
for the volume (default virtio)
- cache (VMs): caching mode (none, writeback,
writethrough; default none)
- dedicatedIOThread (VMs): whether to use a
dedicated I/O thread (true, false; default not
specified)
- fstype (VMs): filesystem type to format
the volume to; empty means to not format
the volume (filesystem must already be present).
fsopts (VMs): options to use for formatting
the filesystem.
- mountopts (VMs): mount options to be used.
- nfsserv (VMs): NFS server for NFS-based PVCs.
- nfsshare (VMs): NFS share name for NFS-based
PVCs. Default to '/'.
Notes:
- A previously declared mount can be overridden
by specifying a later mount with the same
mountpoint.
- A previously declared mount can be removed by
specifying a mount with the same mountpoint
and empty name and type. Example:
--volume=:emptydir:/var/opt/clusterbuster
--volume=::/var/opt/clusterbuster
will result in no mount on /var/opt/clusterbuster
unless later overridden.
- All previously declared mounts can be removed
by specifying a mount with no name, type,
mountpoint, or options. Example:
--volume=::
will remove all mounts from the list.
--workdir=<dir> Use the specified working directory for file I/O
Pod Options:
--container_image=<image>
Image to use (default $container_image).
Does not apply to "classic" or "pause" workloads.
--deployment_type=<pod,deployment,replicaset,vm>
Deploy via individual pods, deployments, replica sets,
or vms (default $deployment_type).
Note that functionality that relies on fixed pod
names or recognition of distinct pods (e. g.
the %i functionality in volumes) will not work
correctly with deployments or replicasets.
--external_sync=host:port
Sync to external host rather than internally
--request=<resource=value>
Resource requests
--limit=<resource=value>
Resource limits
--runtimeclass=[class1,class2...]=class
Run the pods in the designated runtimeclass.
--runtimeclass=vm is a synonym for
--deployment_type=vm.
--kata Synonym for --runtimeclass=${kata_runtime_class}
--tolerate=<key:operator:effect>
Apply the specified tolerations to created pods.
--image_pull_policy=<policy>
Image pull policy (system default)
--node_selector=selector
Annotate pods with the specified node selector
Default $node_selector
Specify empty value to not provide a node selector.
--pod_annotation=[:class:]annotation
Apply the specified annotation to all pods of the
optionally specified class (same meaning as for
--pin_node as above). This may be specified
multiple times.
--headless-services=[0,1]
Use headless services for service creation.
Default ${headless_services}
--liveness-probe=<interval>
Execute a simple liveness probe every <interval>
seconds.
--liveness-probe-sleep=<seconds>
Arrange for the liveness probe to sleep for specified
time.
--privileged-pods=[0,1]
Create pods as privileged (default $create_pods_privileged)
--label=[:class:]label=value
Apply the specified label to all pods of the
optionally specified class (same meaning as for
--pin_node as above). This may be specified
multiple times.
--scheduler=<scheduler>
Use the specified scheduler to schedule pods.
--interface[=:class:]=name[:internal-interface]
Provide the specified network interface to the pod/VM.
Class has the same meaning as for --pin-node.
Internal-interface, if specified, is the name of the
interface inside the pod/VM. Normally this should not
be specified, and the default (net1 for pods, eth1 for
VMs) should be used.
Kata Virtualization Tuning:
--virtiofsd-writeback=[0,1]
Use writeback caching for virtiofsd (default $virtiofsd_writeback).
--virtiofsd-direct=[0,1]
Allow use of direct I/O for virtiofsd (default $virtiofsd_direct).
--virtiofsd-threadpoolsize=n
Use the specified thread pool size for virtiofsd
(default 1).
OpenShift Virtualization Options:
--vm-threads=<value>
Specify the number of threads on each core (default $vm_threads).
--vm-cores=<value>
Specify the number of cores on each socket (default $vm_cores).
--vm-sockets=<value>
Specify the number of sockets (default $vm_sockets).
--vm-memory=<value>
Specify the amount of memory (default $vm_memory).
--vm-grace-period=<value>
Specify the period between when a vm is signaled to
shutdown and the point when KubeVirt will force off
the vm (default $vm_grace_period).
--vm-image=<image_url>
Containerdisk image to use.
Default $vm_image
--vm-migrate=[0,1]
Allow VMs to migrate when evicted rather than be
deleted. Default $vm_evict_migrate.
--vm-run-as-container=[0,1]
Run the workload as a container rather than directly.
--vm-user=<user>
Create the specified user on virtual machines.
Default $vm_user. Empty means no user.
--vm-password=<password>
Create the specified password on virtual machines.
Default $vm_password. Empty means no password.
--vm-ssh-keyfile=file
Inject the public key of the specified key pair into
VMs for log retrieval or other access purposes.
Default none, in which case a temporary key
is generated.
--vm-run-as-root=[0,1]
Run test command as root. Default $vm_run_as_root.
--vm-start-running=[0,1]
Start the VMs in running state (otherwise are started
separately). Default $vm_start_running.
--vm-run_strategy=<strategy>
Specify the desired run strategy for VMs.
Tuning object creation (short equivalents):
--scale-ns=[0,1] Scale up the number of namespaces vs.
create new ones (default 0).
--scale-deployments=[0,1]
Scale up the number of deployments vs.
create new ones (default 1)
--first_deployment=N
Specify the index of the first deployment.
Default is 0 (but see --scale_deployments)
--first_secret=N
Index number of first secret to be created
--first_namespace=N
Index number of first namespace to be created
--pod-prefix=prefix
Prefix all created pods with this prefix.
--sleep=N Number of seconds between object creations
Below options default to sleeptime
--sleep_between_secrets=N
--sleep_between_namespaces=N
--sleep_between_deployments=N
--objs_per_call=N
Number of objects per CLI call. Only objects
within a namespace can be created this way;
to improve creation performance with multiple
namespaces, use --parallel.
Below options all default to objs_per_call
--objs_per_call_secrets=N
--objs_per_call_namespaces=N
--objs_per_call_deployments=N
--parallel=N Number of operations in parallel. Only
operations across namespaces can be
parallelized; to improve performance
within one namespace, use --objs_per_call.
Below options all default to parallel
--parallel_secrets=N
--parallel_namespaces=N
--parallel_deployments=N
--wait_secrets Wait for secrets to be created (default 1)
--pod-start-timeout=<seconds>
Wait specified time for pods to come on line.
Default $default_pod_start_timeout ($default_vm_start_timeout for VMs).
Workload-specific options:
$(_help_options_workloads)
Advanced options (generally not required):
--baseoffset=N Add specified offset to base time
for calculation of start time offset
to correct for clock skew. May be float.
This normally should not be needed, as
ClusterBuster can correct for clock skew
itself.
--podsleep=N Time for pod to sleep before exit
--debug=<opt>
For testing purposes, print debugging information.
Options documented only in code.
--inject_error=<opt>
For testing purposes, inject the specified error
condition (documented only in code).
--force-abort Abort the run on any error.
--preserve-tmpdir
Do not remove the temporary directory at
end of the run
Here is a brief description of all available workloads:
$(_document_workloads)
EOF
}
function help() {
_help_extended "$@" | "${PAGER:-more}"
exit 1
}
function help_extended() {
_help_extended "$@" | "${PAGER:-more}"
exit 1
}
################################################################
# Option processing
################################################################
function set_workload_bytes() {
local sizespec=$1
local -i scale=${2:-1}
if [[ $sizespec = *','* ]] ; then
bytes_transfer=$(parse_size "${sizespec#*,}")
bytes_transfer_max=$(parse_size "${sizespec%%,*}")
if (( bytes_transfer > bytes_transfer_max )) ; then
local -i tmp=$bytes_transfer
bytes_transfer=$bytes_transfer_max
bytes_transfer_max=$tmp
fi
else
bytes_transfer=$((sizespec * scale))
bytes_transfer_max=$((sizespec * scale))
fi
}
function set_runtime() {
local timespec=$1
if [[ $timespec = *','* ]] ; then
workload_run_time=${timespec#*,}
workload_run_time_max=${timespec%%,*}
if (( workload_run_time > workload_run_time_max )) ; then
local -i tmp=$workload_run_time
workload_run_time=$workload_run_time_max
workload_run_time_max=$tmp
fi
else
workload_run_time=$timespec
workload_run_time_max=$timespec
fi
}
function process_pin_node() {
local nodespec=$1
nodespec=${nodespec// /}
[[ -n "$nodespec" ]] || return
if [[ $nodespec = *'='* ]] ; then
local node=${nodespec#*=}
local class=${nodespec%%=*}
class=${class//,/ }
# shellcheck disable=SC2206
local -a classes=($class)
for class in "${classes[@]}" ; do
pin_nodes[$class]="$node"
done
else
pin_nodes[default]="$nodespec"
fi
}
function process_interface() {
local if_spec=$1
if_spec=${if_spec// /}
[[ -n "$if_spec" ]] || return
if [[ $if_spec = *'='* ]] ; then
local interface=${if_spec#*=}
local class=${if_spec%%=*}
class=${class//,/ }
# shellcheck disable=SC2206
local -a classes=($class)
for class in "${classes[@]}" ; do
net_interfaces[$class]="$interface"
done
else
net_interfaces[default]="$if_spec"
fi
}
function process_runtimeclass() {
local runtimespec=$1
if [[ $runtimespec = 'vm' ]] ; then
deployment_type=vm
runtime_class=vm
return
fi
if [[ $runtimespec = 'pod' ]] ; then
runtime_class=
return
fi
runtimespec=${runtimespec// /}
runtime_class=$runtimespec
if [[ -z "$runtimespec" ]] ; then
runtime_classes=()
elif [[ $runtimespec = *'='* ]] ; then
local runtime=${runtimespec#*=}
local class=${runtimespec%%=*}
class=${class//,/ }
# shellcheck disable=SC2206
local -a classes=($class)
for class in "${classes[@]}" ; do
runtime_classes[$class]="$runtime"
done
else
runtime_classes[default]="$runtimespec"
fi
}
function set_metrics_file() {
metrics_file=${1:-}
case "$metrics_file" in
default|1|"$default_metrics_file") metrics_file="$__libdir__/$default_metrics_file" ;;
''|0|none) metrics_file= ;;
*) ;;
esac
}
function inject_error() {
local error="$1"
local condition
local options
IFS='=' read -r condition options <<< "$error"
injected_errors["$condition"]=${options:-SET}
warn "*** Registering error injection '$condition' = '${injected_errors[$condition]}'"
}
function artifact_dirname() {
local basename=${1:-cb-%T}
echo "${basename//%T/$(printf "%($(standard_snapshot_date_format))T" -1)}"
}
function process_option() {
local noptname
local noptname1
local optvalue
read -r noptname1 noptname optvalue <<< "$(parse_option "$1")"
# shellcheck disable=SC2206
# shellcheck disable=SC2119
processed_options+=("--$1")
# shellcheck disable=SC2034
case "$noptname1" in
# Help, verbosity
helpall*) help_extended ;;
helpeverything*) help_extended ;;
help*) help ;;
verbose) verbose=$(bool "$optvalue") ;;
doit) doit=$(bool "$optvalue") ;;
quiet) verbose=$((! $(bool "$optvalue"))) ;;
forceabort*) set -e ;;
preservetmpdir) preserve_tmpdir=$(bool "$optvalue") ;;
# Reporting
artifactdir) artifactdir="$(artifact_dirname "$optvalue")";;
metrics|metricsfile) set_metrics_file "$optvalue" ;;
metricsepoch) metrics_epoch=$optvalue ;;
metricsinterval) metrics_interval=$optvalue ;;
reportformat) report_format=$optvalue ;;
jsonreport) report_format=json ;;
rawreport) report_format=raw ;;
report) report_format=${optvalue:-summary} ;;
verbosereport) report_format=verbose ;;
reportobjectcreation) report_object_creation=$(bool "$optvalue") ;;
prometheussnapshot) take_prometheus_snapshot=$(bool "$optvalue");;
predelay) predelay=$optvalue ;;
postdelay) postdelay=$optvalue ;;
stepinterval) workload_step_interval=$optvalue ;;
timeout) timeout=$optvalue ;;
failurestatus) failure_status=$optvalue ;;
parallellog*) parallel_log_retrieval=$optvalue ;;
retrievesuc*) retrieve_successful_logs=$(bool "$optvalue");;
logsuc*) retrieve_successful_logs=$(bool "$optvalue");;
# Basic options
jobname) job_name=$optvalue ;;
workload) requested_workload=$optvalue ;;
basename) basename=$optvalue ;;
arch) arch=$optvalue ;;
createnamespacesonly) create_namespaces_only=1 ;;
watchdogtimeout) watchdog_timeout=$(parse_size "$optvalue") ;;
# Object definition
workdir) common_workdir=$optvalue ;;
configmapfile) configmap_files+=("$optvalue") ;;
containerimage) container_image=$optvalue ;;
containers) containers_per_pod=$optvalue ;;
containersperpod) containers_per_pod=$optvalue ;;
deploymenttype) deployment_type=$optvalue ;;
deployments|depspername*) deps_per_namespace=$optvalue ;;
exitatend) exit_at_end=$(bool "$optvalue") ;;
imagepullpolicy) image_pull_policy=$optvalue ;;
namespaces) namespaces=$optvalue ;;
nodeselector) node_selector=$optvalue ;;
volume) volumes+=("$optvalue") ;;
processes|processesperpod) processes_per_pod=$optvalue ;;
jobfile) process_job_file "$optvalue" ;;
pinnode) process_pin_node "$optvalue" ;;
interface) process_interface "$optvalue" ;;
replicas) replicas=$optvalue ;;
limit|limits) resource_limits+=("$optvalue") ;;
request|requests) resource_requests+=("$optvalue") ;;
kata) process_runtimeclass "kata" ;;
podannotation) pod_annotations+=("$optvalue") ;;
label|labels) pod_labels+=("$optvalue") ;;
runtimeclass) process_runtimeclass "$optvalue" ;;
uuid) uuid=$optvalue ;;
secrets) secrets=$optvalue ;;
workloadruntime) set_runtime "$optvalue" ;;
workload_size) set_workload_bytes "$optvalue" ;;
targetdatarate) target_data_rate=$(parse_size "$optvalue") ;;
tolerate|toleration) tolerations+=("$optvalue") ;;
dropcache) drop_node_cache=$(bool "$optvalue") ;;
dropallcache) drop_all_node_cache=$(bool "$optvalue") ;;
headlessservices) headless_services=$(bool "$optvalue") ;;
virtiofsdwriteback) virtiofsd_writeback=$(bool "$optvalue") ;;
virtiofsddirect) virtiofsd_direct=$(bool "$optvalue") ;;
virtiofsdthread*) virtiofsd_threadpoolsize=$optvalue ;;
livenessprobeint*) liveness_probe_frequency=$optvalue ;;
livenessprobesleep*) liveness_probe_sleep_time=$optvalue ;;
privilege*) create_pods_privileged=$(bool "$optvalue") ;;
syncinfirst*) sync_in_first_namespace=$(bool "$optvalue") ;;
scheduler) scheduler=$optvalue ;;
affinity)
case "$optvalue" in
1|'') affinity=1 ;;
2|anti) affinity=2 ;;
*) affinity=0 ;;
esac
;;
antiaffinity)
case "$optvalue" in
1|'') affinity=2 ;;
*) affinity=0 ;;
esac
;;
syncaffinity)
case "$optvalue" in
1|'') sync_affinity=1 ;;
2|anti) sync_affinity=2;;
*) sync_affinity=0 ;;
esac
;;
syncantiaffinity)
case "$optvalue" in
1|'') sync_affinity=2 ;;
*) sync_affinity=0 ;;
esac
;;
# vm specs
vmcores) vm_cores=$optvalue ;;
vmsockets) vm_sockets=$optvalue ;;
vmthreads) vm_threads=$optvalue ;;
vmmemory) vm_memory=$optvalue ;;
vmgraceperiod) vm_grace_period=$optvalue ;;
vmimage) vm_image=$optvalue ;;
vmmigrate*) vm_evict_migrate=$(bool "$optvalue") ;;