diff --git a/helm/crabserver/Chart.yaml b/helm/crabserver/Chart.yaml index 5e3d675e4..18c135581 100644 --- a/helm/crabserver/Chart.yaml +++ b/helm/crabserver/Chart.yaml @@ -15,9 +15,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.1 +version: 1.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: "v3.240809" +appVersion: "v3.250930" diff --git a/helm/crabserver/apply_logstash.sh b/helm/crabserver/apply_logstash.sh deleted file mode 100644 index aa6ec8b22..000000000 --- a/helm/crabserver/apply_logstash.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -x -set -euo pipefail -ENV=test -kubectl create configmap logstash-crab --from-file=config/${ENV}/logstash/logstash.conf --from-file config/${ENV}/logstash/logstash.yml --dry-run=client -oyaml | kubectl apply -f - -kubectl delete pod $(kubectl get pod --no-headers -o custom-columns=":metadata.name" | grep logstash) diff --git a/helm/crabserver/config/preprod/logstash/logstash.yml b/helm/crabserver/config/logstash.yml similarity index 100% rename from helm/crabserver/config/preprod/logstash/logstash.yml rename to helm/crabserver/config/logstash.yml diff --git a/helm/crabserver/config/preprod/config.py b/helm/crabserver/config/preprod/config.py index 4e9e23000..a63347110 100644 --- a/helm/crabserver/config/preprod/config.py +++ b/helm/crabserver/config/preprod/config.py @@ -39,7 +39,6 @@ data.loggingLevel = 10 data.loggingFile = '%s/logs/crabserver/CRAB-%s.log' % (__file__.rsplit('/', 4)[0], myhost) data.keptLogDays = 7 -data.mode = "cmsweb-preprod" data.delegateDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cmscrab/CN=(817881|373708)/CN=Robot: cms crab|/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=crabint1/CN=373708/CN=Robot: CMS CRAB Integration 1" data.compatibleClientVersions = ["v3", "development"] diff --git a/helm/crabserver/config/test/logstash/logstash.conf b/helm/crabserver/config/preprod/logstash.conf similarity index 88% rename from helm/crabserver/config/test/logstash/logstash.conf rename to helm/crabserver/config/preprod/logstash.conf index e47fb7852..6326ce849 100644 --- a/helm/crabserver/config/test/logstash/logstash.conf +++ b/helm/crabserver/config/preprod/logstash.conf @@ -169,15 +169,8 @@ filter { "type" => "crabhttpcall" } } - # parse log path to get pod name - # /var/log/pods/crab_crabserver-test-5f9bd5f58d-5nfhc_5366ebdb-ee96-4641-8643-f3bbe63f808f/crabserver/0.log grok { - match => { "[log][file][path]" => '/var/log/pods/crab_%{DATA:pod_name}_%{DATA}/%{GREEDYDATA}' } - } - - grok { - # [07/Dec/2022:20:06:47] crabserver-56c69685b6-477jr 188.184.91.103:41924 "GET /crabserver/devthree/info HTTP/1.1" 200 OK [data: 9354 in 103 out 4870 us ] [auth: ok "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=tseethon/CN=856006/CN=Thanayut Seethongchuen" "" ] [ref: "https://cmsweb-test12.cern.ch" "Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0" ] - Podname=crabserver-6cf49bcfff-bxvb6 Type=cherrypylog - match => { "message" => '\[%{NOTSPACE:timestamp_temp}\] %{DATA:backend} %{IPORHOST:clientip}(|:%{NUMBER}) "%{WORD:method} %{NOTSPACE:request} %{DATA:httpversion}" %{NUMBER:code:int} %{DATA} \[data: (-|%{NUMBER:bytes_sent:int}) in (-|%{NUMBER:bytes_received:int}) out (-|%{NUMBER:time_spent_ms:int}) us \] \[auth: %{DATA} "%{DATA:dn}".*\] \[ref: "%{DATA}.*" "%{DATA:client}" \]' } + match => { "message" => '\[%{NOTSPACE:timestamp_temp}\] %{DATA:backend} %{IPORHOST:clientip} "%{WORD:method} %{NOTSPACE:request} %{DATA:httpversion}" %{NUMBER:code:int} %{DATA} \[data: (-|%{NUMBER:bytes_sent:int}) in (-|%{NUMBER:bytes_received:int}) out (-|%{NUMBER:time_spent_ms:int}) us \] \[auth: %{DATA} "%{DATA:dn}".*\] \[ref: "%{DATA}.*" "%{DATA:client}" \]' } } grok { match => { "request" => '/%{WORD:system}%{UNIXPATH:uri_path}%{URIPARAM:uri_params}?' } @@ -208,15 +201,9 @@ filter { "type" => "crabrest" } } - # parse log path to get pod name - # /var/log/pods/crab_crabserver-test-5f9bd5f58d-5nfhc_5366ebdb-ee96-4641-8643-f3bbe63f808f/crabserver/0.log - grok { - match => { "[log][file][path]" => '/var/log/pods/crab_%{DATA:pod_name}_%{DATA}/%{GREEDYDATA}' } - } - if ( [message] =~ /.*MeasureTime:seconds.*/ ) { grok { - match => { "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:trace_id}:%{NOTSPACE:log_level}:%{NOTSPACE}:MeasureTime:seconds - modulename=%{NOTSPACE:modulename} label='%{NOTSPACE:label}' tot=%{NUMBER:tot:float} proc=%{NUMBER:proc:float} thread=%{NUMBER:thread:float}" } + match => { "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:trace_id}:%{NOTSPACE:log_level}:%{NOTSPACE}:MeasureTime:seconds - modulename=%{NOTSPACE:modulename} label='%{NOTSPACE:label}' tot=%{NUMBER:tot:float} proc=%{NUMBER:total:float} thread=%{NUMBER:thread:float}" } add_field => {"log_type" => "measure_time"} } } else if ( [message] =~ /.*MeasureSize:bytes.*/ ) { diff --git a/helm/crabserver/config/prod/config.py b/helm/crabserver/config/prod/config.py index 31f098cf8..6e7dcb414 100644 --- a/helm/crabserver/config/prod/config.py +++ b/helm/crabserver/config/prod/config.py @@ -37,6 +37,7 @@ data.backend = 'oracle' data.db = 'CRABServerAuth.dbconfig' data.s3 = 'CRABServerAuth.s3' +data.s3_bucket = 'crabcache_prod' data.workflowManager = 'HTCondorDataWorkflow' data.extconfigurl = 'http://gitlab.cern.ch/crab3/CRAB3ServerConfig/raw/master/cmsweb-rest-config.json' @@ -44,6 +45,6 @@ data.loggingLevel = 10 data.loggingFile = '%s/logs/crabserver/CRAB-%s.log' % (__file__.rsplit('/', 4)[0], myhost) data.keptLogDays = 7 -data.mode = "cmsweb-prod" -data.delegateDN = "/DC=ch/DC=cern/OU=computers/CN=crab-(preprod|prod)-tw(01|02).cern.ch|/DC=ch/DC=cern/OU=computers/CN=crab-dev-tw(01|02|03|04).cern.ch|/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cmscrab/CN=(817881|373708)/CN=Robot: cms crab|/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=crabint1/CN=373708/CN=Robot: CMS CRAB Integration 1" +data.delegateDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cmscrab/CN=(817881|373708)/CN=Robot: cms crab|/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=crabint1/CN=373708/CN=Robot: CMS CRAB Integration 1" +data.compatibleClientVersions = ["v3", "development"] diff --git a/helm/crabserver/config/preprod/logstash/logstash.conf b/helm/crabserver/config/prod/logstash.conf similarity index 88% rename from helm/crabserver/config/preprod/logstash/logstash.conf rename to helm/crabserver/config/prod/logstash.conf index e47fb7852..6326ce849 100644 --- a/helm/crabserver/config/preprod/logstash/logstash.conf +++ b/helm/crabserver/config/prod/logstash.conf @@ -169,15 +169,8 @@ filter { "type" => "crabhttpcall" } } - # parse log path to get pod name - # /var/log/pods/crab_crabserver-test-5f9bd5f58d-5nfhc_5366ebdb-ee96-4641-8643-f3bbe63f808f/crabserver/0.log grok { - match => { "[log][file][path]" => '/var/log/pods/crab_%{DATA:pod_name}_%{DATA}/%{GREEDYDATA}' } - } - - grok { - # [07/Dec/2022:20:06:47] crabserver-56c69685b6-477jr 188.184.91.103:41924 "GET /crabserver/devthree/info HTTP/1.1" 200 OK [data: 9354 in 103 out 4870 us ] [auth: ok "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=tseethon/CN=856006/CN=Thanayut Seethongchuen" "" ] [ref: "https://cmsweb-test12.cern.ch" "Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0" ] - Podname=crabserver-6cf49bcfff-bxvb6 Type=cherrypylog - match => { "message" => '\[%{NOTSPACE:timestamp_temp}\] %{DATA:backend} %{IPORHOST:clientip}(|:%{NUMBER}) "%{WORD:method} %{NOTSPACE:request} %{DATA:httpversion}" %{NUMBER:code:int} %{DATA} \[data: (-|%{NUMBER:bytes_sent:int}) in (-|%{NUMBER:bytes_received:int}) out (-|%{NUMBER:time_spent_ms:int}) us \] \[auth: %{DATA} "%{DATA:dn}".*\] \[ref: "%{DATA}.*" "%{DATA:client}" \]' } + match => { "message" => '\[%{NOTSPACE:timestamp_temp}\] %{DATA:backend} %{IPORHOST:clientip} "%{WORD:method} %{NOTSPACE:request} %{DATA:httpversion}" %{NUMBER:code:int} %{DATA} \[data: (-|%{NUMBER:bytes_sent:int}) in (-|%{NUMBER:bytes_received:int}) out (-|%{NUMBER:time_spent_ms:int}) us \] \[auth: %{DATA} "%{DATA:dn}".*\] \[ref: "%{DATA}.*" "%{DATA:client}" \]' } } grok { match => { "request" => '/%{WORD:system}%{UNIXPATH:uri_path}%{URIPARAM:uri_params}?' } @@ -208,15 +201,9 @@ filter { "type" => "crabrest" } } - # parse log path to get pod name - # /var/log/pods/crab_crabserver-test-5f9bd5f58d-5nfhc_5366ebdb-ee96-4641-8643-f3bbe63f808f/crabserver/0.log - grok { - match => { "[log][file][path]" => '/var/log/pods/crab_%{DATA:pod_name}_%{DATA}/%{GREEDYDATA}' } - } - if ( [message] =~ /.*MeasureTime:seconds.*/ ) { grok { - match => { "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:trace_id}:%{NOTSPACE:log_level}:%{NOTSPACE}:MeasureTime:seconds - modulename=%{NOTSPACE:modulename} label='%{NOTSPACE:label}' tot=%{NUMBER:tot:float} proc=%{NUMBER:proc:float} thread=%{NUMBER:thread:float}" } + match => { "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:trace_id}:%{NOTSPACE:log_level}:%{NOTSPACE}:MeasureTime:seconds - modulename=%{NOTSPACE:modulename} label='%{NOTSPACE:label}' tot=%{NUMBER:tot:float} proc=%{NUMBER:total:float} thread=%{NUMBER:thread:float}" } add_field => {"log_type" => "measure_time"} } } else if ( [message] =~ /.*MeasureSize:bytes.*/ ) { diff --git a/helm/crabserver/config/prod/logstash/logstash.conf b/helm/crabserver/config/prod/logstash/logstash.conf deleted file mode 100644 index e47fb7852..000000000 --- a/helm/crabserver/config/prod/logstash/logstash.conf +++ /dev/null @@ -1,272 +0,0 @@ -# Migrated from: https://github.com/dmwm/CRABServer/blob/master/src/script/Monitor/logstash/crabtaskworker.conf - -input { beats { port => 5044 } } - -filter { - # whitelisted tags - if "crabtaskworker" not in [tags] and "crabpublisher" not in [tags] and "crabhttpcall" not in [tags] and "crabrest" not in [tags] { drop { } } - - mutate { - # ---- Common fields ---- [metadata] - add_field => { - "cmsweb_cluster" => "${CMSWEB_CLUSTER:NA}" - "cmsweb_env" => "${CMSWEB_ENV:NA}" - "filebeat_id" => "%{[agent][id]}" - "filebeat_name" => "%{[agent][name]}" - "filebeat_version" => "%{[agent][version]}" - "hostname" => "%{[host][name]}" - "log_file" => "%{[log][file][path]}" - } - } - - if "crabtaskworker" in [tags] { - # ------------------------------------------------------------------------------------------------------------------------------------------------------------------ - # TASKWORKER - mutate { - # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers! - replace => { - "producer" => "crab" - "type" => "crabtaskworker" - } - } - if "completed in" in [message] { - grok { - match => { - "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:DEBUG:Worker,%{INT}:Process-%{NOTSPACE:slaveID}: %{NOTSPACE:workType} work on %{NOTSPACE:taskName} completed in %{NUMBER:completionTime:int} seconds" - } - add_field => {"log_type" => "work_on_task_completed"} - overwrite => ["message"] - } - - } else if ( [message] =~ /.*Starting.*at.*on.*/ ) { - grok { - match => { - "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:DEBUG:Worker,%{INT}:Process-%{NOTSPACE:slaveID}: Starting on %{NOTSPACE:taskName}" - } - add_field => {"log_type" => "start_new_task"} - overwrite => ["message"] - } - } else if "TWSTART" in [message] { - grok { - match => { - "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:MasterWorker,%{INT}:TWSTART: %{GREEDYDATA:tw_json_data}" - } - add_field => {"log_type" => "tw_config_data"} - overwrite => ["message"] - } - if [tw_json_data] { - json { - source => "tw_json_data" - target => "tw_json_data" - } - } - - } else if ( [message] =~ /.*Finished.*object at .*/ ) { - grok { - match => { - "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:INFO:Handler,%{INT}:Finished on %{NOTSPACE:taskName} in %{NUMBER:completionTime:int} seconds" - } - add_field => {"log_type" => "action_on_task_finished"} - overwrite => ["message"] - } - } else { drop {} } - - date { - # Since date string does not provide timezone, we need to specify it explicitly - match => ["timestamp_temp", "YYYY-MM-dd HH:mm:ss,SSS"] - timezone => "Europe/Zurich" - target => "rec_timestamp_str" - } - # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - } else if "crabpublisher" in [tags] { - # ------------------------------------------------------------------------------------------------------------------------------------------------------------------ - # CRABPUBLISHER - mutate { - # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers! - replace => { - "producer" => "crab" - "type" => "crabpublisher" - } - } - - if "PUBSTART" in [message] { - grok { - match => { - "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:PUBSTART:%{GREEDYDATA:pb_json_data}" - } - add_field => {"log_type" => "publisher_config_data"} - overwrite => ["message"] - } - if [pb_json_data] { - json{ - source => "pb_json_data" - target => "pb_json_data" - } - } - - } else if ( [message] =~ /.*blocks failed.*files.*/ ) { - grok { - match => { - "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:Taskname %{NOTSPACE:taskName} : %{INT:blocks} blocks failed for a total of %{INT:files} files" - } - add_field => {"log_type" => "failed_publication"} - overwrite => ["message"] - } - - } else if ( [message] =~ /.*Published.*files in.*/ ) { - grok { - match => { - "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:Taskname %{NOTSPACE:taskName} is %{NOTSPACE:publicationResult}. Published %{INT:filesPublished} files in %{INT:blocks} blocks." - } - add_field => {"log_type" => "successful_publication"} - overwrite => ["message"] - } - - } else if ( [message] =~ /.*Exception.*TaskPublish.*/ ) { - grok { - match => { - "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:%{GREEDYDATA:exceptionHandled}" - } - add_field => {"log_type" => "publication_error"} - overwrite => ["message"] - } - - # This filter is currently used for production publisher. After we deploy preprod to prod, this filter can be dropped - } else if "DEBUG:master" in [message] { - grok { - match => { - "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:master:%{SPACE}%{INT:acquiredFiles}%{SPACE}:%{SPACE}%{NOTSPACE:taskName}" - } - add_field => {"log_type" => "acquired_files"} - overwrite => ["message"] - } - - # this filter mathces changes in #6861, which is already in preprod and dev - } else if "acquired_files" in [message] { - grok { - match => { - "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:logMsg}:PublisherMaster,%{INT}:acquired_files:%{SPACE}%{NOTSPACE:acquiredFilesStatus}%{SPACE}%{INT:acquiredFiles}%{SPACE}:%{SPACE}%{NOTSPACE:taskName}" - } - add_field => {"log_type" => "acquired_files_status"} - overwrite => ["message"] - } - - } else { drop { } } - date { - # Since date string does not provide timezone, we need to specify it explicitly - match => ["timestamp_temp", "YYYY-MM-dd HH:mm:ss,SSS"] - timezone => "Europe/Zurich" - target => "rec_timestamp_str" - } - # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - } else if "crabhttpcall" in [tags] { - # ------------------------------------------------------------------------------------------------------------------------------------------------------------------ - # CRABHTTPCALL (previously crabserver) - mutate { - # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers! - replace => { - "producer" => "crab" - "type" => "crabhttpcall" - } - } - # parse log path to get pod name - # /var/log/pods/crab_crabserver-test-5f9bd5f58d-5nfhc_5366ebdb-ee96-4641-8643-f3bbe63f808f/crabserver/0.log - grok { - match => { "[log][file][path]" => '/var/log/pods/crab_%{DATA:pod_name}_%{DATA}/%{GREEDYDATA}' } - } - - grok { - # [07/Dec/2022:20:06:47] crabserver-56c69685b6-477jr 188.184.91.103:41924 "GET /crabserver/devthree/info HTTP/1.1" 200 OK [data: 9354 in 103 out 4870 us ] [auth: ok "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=tseethon/CN=856006/CN=Thanayut Seethongchuen" "" ] [ref: "https://cmsweb-test12.cern.ch" "Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0" ] - Podname=crabserver-6cf49bcfff-bxvb6 Type=cherrypylog - match => { "message" => '\[%{NOTSPACE:timestamp_temp}\] %{DATA:backend} %{IPORHOST:clientip}(|:%{NUMBER}) "%{WORD:method} %{NOTSPACE:request} %{DATA:httpversion}" %{NUMBER:code:int} %{DATA} \[data: (-|%{NUMBER:bytes_sent:int}) in (-|%{NUMBER:bytes_received:int}) out (-|%{NUMBER:time_spent_ms:int}) us \] \[auth: %{DATA} "%{DATA:dn}".*\] \[ref: "%{DATA}.*" "%{DATA:client}" \]' } - } - grok { - match => { "request" => '/%{WORD:system}%{UNIXPATH:uri_path}%{URIPARAM:uri_params}?' } - } - if ![uri_params] { - ruby { code => "event.set('uri_params','')" } - } - grok { match => { "uri_path" => '/.*/%{DATA:api}$' } } - if [client] { - grok { match => { "client" => '%{DATA:client_name}/%{DATA:client_version}$' } } - } - # !ATTENTION! Different date-time format than publisher and TW, means that special for crabserver. - date { - # Since date string does not provide timezone, we need to specify it explicitly - match => [ "timestamp_temp", "dd/MMM/yyyy:HH:mm:ss" ] - timezone => "Europe/Zurich" - target => "rec_timestamp_str" - } - mutate { gsub => [ "dn","/CN=\d+","" ] } - # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - } else if "crabrest" in [tags] { - # ------------------------------------------------------------------------------------------------------------------------------------------------------------------ - # CRABLOG - mutate { - # MONIT mandatory fields. Be aware that you cannot use any of these names in grok/mutate parsers! - replace => { - "producer" => "crab" - "type" => "crabrest" - } - } - # parse log path to get pod name - # /var/log/pods/crab_crabserver-test-5f9bd5f58d-5nfhc_5366ebdb-ee96-4641-8643-f3bbe63f808f/crabserver/0.log - grok { - match => { "[log][file][path]" => '/var/log/pods/crab_%{DATA:pod_name}_%{DATA}/%{GREEDYDATA}' } - } - - if ( [message] =~ /.*MeasureTime:seconds.*/ ) { - grok { - match => { "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:trace_id}:%{NOTSPACE:log_level}:%{NOTSPACE}:MeasureTime:seconds - modulename=%{NOTSPACE:modulename} label='%{NOTSPACE:label}' tot=%{NUMBER:tot:float} proc=%{NUMBER:proc:float} thread=%{NUMBER:thread:float}" } - add_field => {"log_type" => "measure_time"} - } - } else if ( [message] =~ /.*MeasureSize:bytes.*/ ) { - grok { - match => { "message" => "%{TIMESTAMP_ISO8601:timestamp_temp}:%{NOTSPACE:trace_id}:%{NOTSPACE:log_level}:%{NOTSPACE}:MeasureSize:bytes - modulename=%{NOTSPACE:modulename} label='%{NOTSPACE:label}' obj_size=%{INT:obj_size:int} get_size_time=%{NUMBER:get_size_time:float}" } - add_field => {"log_type" => "measure_size"} - } - } - date { - # Since date string does not provide timezone, we need to specify it explicitly - match => ["timestamp_temp", "YYYY-MM-dd HH:mm:ss,SSS"] - timezone => "Europe/Zurich" - target => "rec_timestamp_str" - } - # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - } - - # ---- Common modifications ---- - if "_grokparsefailure" in [tags] { drop { } } - - # Date operations - ruby { - # "metadata.timestamp"(date in ES) equals to record_time(int in ES) which we need for latency calculation - # "producer_time" is filebeat process time stamp. - code => "event.set('timestamp', (event.get('rec_timestamp_str').to_f * 1000).to_i) - event.set('record_time', (event.get('rec_timestamp_str').to_f * 1000).to_i) - event.set('producer_time', (event.get('@timestamp').to_f * 1000).to_i) - " - } - - # remove quotes from message entry since it will break the JSON - mutate { gsub => [ "message", "\n", "", "message", "\"", ""] } - - # remove undesired fields and objects - mutate { remove_field => ["@timestamp", "@version", "timestamp_temp", "agent", "log", "input", "tags", "ecs", "host", "cloud", "event" ] } - -} - -output { - if [type] in ["crabtaskworker", "crabpublisher", "crabhttpcall", "crabrest"] { - http { - http_method => "post" - url => "http://monit-logs.cern.ch:10012/" - content_type => "application/json; charset=UTF-8" - format => "json_batch" - socket_timeout => 60 - connect_timeout => 60 - } - } else { - # For debugging, please keep it to catch problems - stdout { codec => json } - } -} diff --git a/helm/crabserver/config/prod/logstash/logstash.yml b/helm/crabserver/config/prod/logstash/logstash.yml deleted file mode 100644 index c826af0ea..000000000 --- a/helm/crabserver/config/prod/logstash/logstash.yml +++ /dev/null @@ -1,7 +0,0 @@ -http.host: "0.0.0.0" -path.config: /usr/share/logstash/pipeline -# disable connection to ES -xpack.monitoring.enabled: false -pipeline.workers: 16 -pipeline.batch.size: 256 -pipeline.batch.delay: 200 diff --git a/helm/crabserver/config/test/config.py b/helm/crabserver/config/test/config.py index 849593d88..44d4792e6 100644 --- a/helm/crabserver/config/test/config.py +++ b/helm/crabserver/config/test/config.py @@ -39,7 +39,6 @@ data.loggingLevel = 10 data.loggingFile = '%s/logs/crabserver/CRAB-%s.log' % (__file__.rsplit('/', 4)[0], myhost) data.keptLogDays = 7 -data.mode = "cmsweb-test" data.enableQueryLoadAllRows = os.getenv('CRABSERVER_ENABLE_QUERY_LOAD_ALL_ROWS', 'True').lower() in ('true', '1', 't') diff --git a/helm/crabserver/config/test/logstash/logstash.yml b/helm/crabserver/config/test/logstash/logstash.yml deleted file mode 100644 index c826af0ea..000000000 --- a/helm/crabserver/config/test/logstash/logstash.yml +++ /dev/null @@ -1,7 +0,0 @@ -http.host: "0.0.0.0" -path.config: /usr/share/logstash/pipeline -# disable connection to ES -xpack.monitoring.enabled: false -pipeline.workers: 16 -pipeline.batch.size: 256 -pipeline.batch.delay: 200 diff --git a/helm/crabserver/deploy.sh b/helm/crabserver/deploy.sh deleted file mode 100755 index 1c88a49d7..000000000 --- a/helm/crabserver/deploy.sh +++ /dev/null @@ -1,41 +0,0 @@ -#! /usr/bin/env bash -set -euo pipefail - -ENVNAME=$1 - -# hash table of clusters nickname-name: -declare -A cluster_map=([prod]=prod) -cluster_map[prod]=k8s-prodsrv -cluster_map[preprod]=k8s-prodsrv-v1.22.9 -cluster_map[testbed]=testbed -cluster_map[test2]=test2 -cluster_map[test11]=test11 -cluster_map[test12]=test12 -cluster_map[test14]=test14 - -declare -A valuefile_map=([prod]=prod) -valuefile_map[prod]=values-prod.yaml -valuefile_map[preprod]=values-preprod.yaml -valuefile_map[testbed]=values-testbed.yaml -valuefile_map[test2]=values-testx.yaml -valuefile_map[test11]=values-testx.yaml -valuefile_map[test12]=values-testx.yaml -valuefile_map[test14]=values-testx.yaml - -if [[ $# -ne 1 ]]; then - echo "Usage: deploy.sh ENVNAME" - echo " ENVNAME=(prod|preprod|testbed|test2|test11|test12|test14)" - exit 1 -fi -desired_cluster="${cluster_map[$ENVNAME]}" -valuefile="${valuefile_map[$ENVNAME]}" - -# make sure that your current context points to the desired cluster -current_cluster=$(kubectl config view -o json | jq '.["current-context"] as $context | .["contexts"][] | select(.name | contains($context))| .context.cluster') -set -x -if [[ $current_cluster =~ $desired_cluster ]]; then - echo "deploying to $desired_cluster"; - helm template crabserver . -f values.yaml -f "${valuefile}" | kubectl -n crab apply -f - -else - echo "Error: wrong cluster. your are connected to $current_cluster"; -fi diff --git a/helm/crabserver/templates/ci-serviceaccount-secret.yaml b/helm/crabserver/templates/ci-serviceaccount-secret.yaml deleted file mode 100644 index fa5adccf9..000000000 --- a/helm/crabserver/templates/ci-serviceaccount-secret.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if .Values.ciServiceAccount.enabled -}} -apiVersion: v1 -kind: Secret -metadata: - annotations: - kubernetes.io/service-account.name: crab-gitlab-ci - name: crab-gitlab-ci-secret - labels: - {{- include "crabserver.labels" . | nindent 4 }} -type: kubernetes.io/service-account-token -{{- end }} \ No newline at end of file diff --git a/helm/crabserver/templates/configMap.yaml b/helm/crabserver/templates/configMap.yaml index 3ddc639d1..bcbf15f67 100644 --- a/helm/crabserver/templates/configMap.yaml +++ b/helm/crabserver/templates/configMap.yaml @@ -5,18 +5,16 @@ apiVersion: v1 metadata: name: crabserver labels: + env: k8s-{{$environment}} app: crabserver namespace: crab data: entrypoint.sh: | {{ (.Files.Get "scripts/entrypoint.sh" | indent 4) }} config.py: | -{{- if (eq (toString $environment) "prod") }} -{{ (.Files.Get "config/prod/config.py" | indent 4) }} -{{- else if (eq (toString $environment) "preprod")}} -{{ (.Files.Get "config/preprod/config.py" | indent 4) }} +{{- if .Values.crabserver.configPy }} +{{ .Values.crabserver.configPy | nindent 4 }} {{- else }} -{{ (.Files.Get "config/test/config.py" | indent 4) }} +{{ (.Files.Get (printf "config/%s/config.py" $environment) | indent 4) }} {{- end }} - {{- end }} diff --git a/helm/crabserver/templates/deployment.yaml b/helm/crabserver/templates/deployment.yaml index 478ca1ed3..09ae62c96 100644 --- a/helm/crabserver/templates/deployment.yaml +++ b/helm/crabserver/templates/deployment.yaml @@ -5,17 +5,12 @@ kind: Deployment metadata: name: crabserver labels: + env: k8s-{{.Values.environment}} app: crabserver namespace: crab spec: {{- if not .Values.autoscaling.enabled }} - {{- if (eq (toString $environment) "prod") }} - replicas: {{ .Values.replicaCount.prod }} - {{- else if (eq (toString $environment) "preprod")}} - replicas: {{ .Values.replicaCount.preprod }} - {{- else }} - replicas: {{ .Values.replicaCount.test }} - {{- end }} + replicas: {{ .Values.replicaCount }} {{- end }} selector: matchLabels: @@ -28,7 +23,7 @@ spec: {{- end }} labels: app: crabserver - env: k8s-{{.Values.environment}} + env: k8s-{{$environment}} spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: @@ -62,21 +57,9 @@ spec: env: {{- toYaml .Values.image.env | nindent 10 }} livenessProbe: - {{- if eq (toString $environment) "prod" }} {{- toYaml .Values.livenessProbe | nindent 12 }} - {{- else if eq (toString $environment) "preprod" }} - {{- toYaml .Values.livenessProbePreProd | nindent 12 }} - {{- else }} - {{- toYaml .Values.livenessProbeTest | nindent 12 }} - {{- end }} readinessProbe: - {{- if eq (toString $environment) "prod" }} {{- toYaml .Values.readinessProbe | nindent 12 }} - {{- else if eq (toString $environment) "preprod" }} - {{- toYaml .Values.readinessProbePreProd | nindent 12 }} - {{- else }} - {{- toYaml .Values.readinessProbeTest | nindent 12 }} - {{- end }} volumeMounts: - name: proxy-secrets mountPath: /etc/proxy diff --git a/helm/crabserver/templates/filebeat-configmap.yaml b/helm/crabserver/templates/filebeat-configmap.yaml index c84c6d60f..24e48fef6 100644 --- a/helm/crabserver/templates/filebeat-configmap.yaml +++ b/helm/crabserver/templates/filebeat-configmap.yaml @@ -26,7 +26,7 @@ data: tags: - "crabrest" output.logstash: - hosts: ["logstash-crab:5044"] + hosts: ["logstash:5044"] compression_level: 3 bulk_max_size: 4096 queue.mem: diff --git a/helm/crabserver/templates/filebeat-daemonset.yaml b/helm/crabserver/templates/filebeat-daemonset.yaml index fa628947f..e92d3a64e 100644 --- a/helm/crabserver/templates/filebeat-daemonset.yaml +++ b/helm/crabserver/templates/filebeat-daemonset.yaml @@ -61,4 +61,4 @@ spec: name: filebeat-crab-config name: config -{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/crabserver/templates/ingress.yaml b/helm/crabserver/templates/ingress.yaml deleted file mode 100644 index 5cb99ced2..000000000 --- a/helm/crabserver/templates/ingress.yaml +++ /dev/null @@ -1,41 +0,0 @@ -{{- if and (.Values.enabled) (.Values.ingress.enabled) -}} -{{- $fullName := include "crabserver.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "crabserver.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ . }} - backend: - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} diff --git a/helm/crabserver/templates/logstash-config-configmap.yml b/helm/crabserver/templates/logstash-config-configmap.yml new file mode 100644 index 000000000..39fc0afc7 --- /dev/null +++ b/helm/crabserver/templates/logstash-config-configmap.yml @@ -0,0 +1,14 @@ +{{- $environment := .Values.environment | default dict }} +{{- if .Values.logPipeline.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + k8s-app: logstash-config + name: logstash-config + namespace: {{ .Values.logPipeline.logstashNamespace }} +data: + # logstash config + logstash.yml: | +{{ (.Files.Get "config/logstash.yml" | indent 4) }} +{{- end }} diff --git a/helm/crabserver/templates/logstash-configmap.yaml b/helm/crabserver/templates/logstash-configmap.yaml deleted file mode 100644 index a61c8e049..000000000 --- a/helm/crabserver/templates/logstash-configmap.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{- $environment := .Values.environment | default dict }} -{{- if .Values.logPipeline.enabled -}} -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - k8s-app: logstash - name: logstash-crab -data: - # logstash config - logstash.yml: | -{{- if (eq (toString $environment) "prod") }} -{{ (.Files.Get "config/prod/logstash/logstash.yml" | indent 4) }} -{{- else if (eq (toString $environment) "preprod")}} -{{ (.Files.Get "config/preprod/logstash/logstash.yml" | indent 4) }} -{{- else }} -{{ (.Files.Get "config/test/logstash/logstash.yml" | indent 4) }} -{{- end }} - # pipeline config - logstash.conf: | -{{- if (eq (toString $environment) "prod") }} -{{ (.Files.Get "config/prod/logstash/logstash.conf" | indent 4) }} -{{- else if (eq (toString $environment) "preprod")}} -{{ (.Files.Get "config/preprod/logstash/logstash.conf" | indent 4) }} -{{- else }} -{{ (.Files.Get "config/test/logstash/logstash.conf" | indent 4) }} -{{- end }} - -{{- end }} diff --git a/helm/crabserver/templates/logstash-deployment.yaml b/helm/crabserver/templates/logstash-deployment.yaml deleted file mode 100644 index ad227f801..000000000 --- a/helm/crabserver/templates/logstash-deployment.yaml +++ /dev/null @@ -1,82 +0,0 @@ -{{- $environment := .Values.environment | default dict }} -{{- if .Values.logPipeline.enabled -}} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: logstash-crab - labels: - app: logstash-crab -spec: - selector: - matchLabels: - app: logstash-crab - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - app: logstash-crab - spec: - containers: - - image: docker.elastic.co/logstash/logstash:8.1.3 - name: logstash - env: - - name: CMSWEB_CLUSTER - {{- if (eq (toString $environment) "prod") }} - value: prod - {{- else if (eq (toString $environment) "preprod")}} - value: testbed - {{- else }} - value: dev - {{- end }} - - name: CMSWEB_ENV - {{- if (eq (toString $environment) "prod") }} - value: prod - {{- else if (eq (toString $environment) "preprod")}} - value: testbed - {{- else }} - value: dev - {{- end }} - - name: LS_JAVA_OPTS - value: "-Xms256m -Xmx4g" - ports: - - name: logstash - containerPort: 5044 - protocol: TCP - volumeMounts: - - name: config - mountPath: /usr/share/logstash/config/logstash.yml - subPath: logstash.yml - readOnly: true - - name: pipeline - mountPath: /usr/share/logstash/pipeline - readOnly: true - args: - - -f - - /usr/share/logstash/pipeline/logstash.conf - - -e - #- --log.level=debug # in k8s, output stdout prints only in debug - resources: - requests: - memory: 512Mi - cpu: "400m" - limits: - memory: 5Gi - cpu: "4000m" - volumes: - - name: pipeline - configMap: - name: logstash-crab - items: - - key: logstash.conf - path: logstash.conf - - name: config - configMap: - name: logstash-crab - items: - - key: logstash.yml - path: logstash.yml - -{{- end }} diff --git a/helm/crabserver/templates/logstash-pipeline-configmap.yml b/helm/crabserver/templates/logstash-pipeline-configmap.yml new file mode 100644 index 000000000..1e502c80f --- /dev/null +++ b/helm/crabserver/templates/logstash-pipeline-configmap.yml @@ -0,0 +1,14 @@ +{{- $environment := .Values.environment | default dict }} +{{- if .Values.logPipeline.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + k8s-app: logstash-pipeline-crab + name: logstash-pipeline-crab + namespace: {{ .Values.logPipeline.logstashNamespace }} +data: + # pipeline config + logstash.conf: | +{{ (.Files.Get (printf "config/%s/logstash.conf" $environment) | indent 4) }} +{{- end }} diff --git a/helm/crabserver/templates/logstash-service.yaml b/helm/crabserver/templates/logstash-service.yaml deleted file mode 100644 index b754f2b09..000000000 --- a/helm/crabserver/templates/logstash-service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.logPipeline.enabled -}} -apiVersion: v1 -kind: Service -metadata: - name: logstash-crab - labels: - app: logstash-crab -spec: - selector: - app: logstash-crab - type: NodePort - ports: - - nodePort: 30044 - targetPort: 5044 - protocol: TCP - port: 5044 -{{- end }} diff --git a/helm/crabserver/templates/service.yaml b/helm/crabserver/templates/service.yaml index 8022584d8..d5bc803d5 100644 --- a/helm/crabserver/templates/service.yaml +++ b/helm/crabserver/templates/service.yaml @@ -2,6 +2,9 @@ kind: Service apiVersion: v1 metadata: + labels: + env: k8s-{{.Values.environment}} + app: crabserver name: crabserver namespace: crab spec: diff --git a/helm/crabserver/values-canary.yaml b/helm/crabserver/values-canary.yaml deleted file mode 100644 index 7716ca584..000000000 --- a/helm/crabserver/values-canary.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -# Deploying canary together with normal deployment -environment: "preprod" - -image: - tag: "v3.240904-stable" - -canary: - enabled: true - replicaCount: 1 - image: - tag: "v3.240904-stable" diff --git a/helm/crabserver/values-clusteradmin.yaml b/helm/crabserver/values-clusteradmin.yaml deleted file mode 100644 index 2890bfedf..000000000 --- a/helm/crabserver/values-clusteradmin.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -enabled: true -environment: "preprod" - -crabserver: - image: - tag: "v3.240904-stable" - canary: - enabled: false - -ciServiceAccount: - enabled: true -logPipeline: - enabled: true diff --git a/helm/crabserver/values-genall.yaml b/helm/crabserver/values-genall.yaml deleted file mode 100644 index e8aaa1350..000000000 --- a/helm/crabserver/values-genall.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -# For debugging only. Generate all resources in this chart. -enabled: true -environment: "test" - -crabserver: - image: - tag: "pypi-test12-manual1" - canary: - enabled: true - replicaCount: 1 - image: - tag: "pypi-test12-manual1" - -ciServiceAccount: - enabled: true -logPipeline: - enabled: true diff --git a/helm/crabserver/values-logpipeline.yaml b/helm/crabserver/values-logpipeline.yaml deleted file mode 100644 index 4924616eb..000000000 --- a/helm/crabserver/values-logpipeline.yaml +++ /dev/null @@ -1,6 +0,0 @@ -environment: "test" - -enabled: false - -logPipeline: - enabled: true diff --git a/helm/crabserver/values-preprod.yaml b/helm/crabserver/values-preprod.yaml new file mode 100644 index 000000000..122132db6 --- /dev/null +++ b/helm/crabserver/values-preprod.yaml @@ -0,0 +1,34 @@ +environment: "preprod" + +image: + tag: "v3-250929-1" + +ciServiceAccount: + enabled: false + + +livenessProbe: + exec: + command: + - /bin/sh + - -c + - | + cmsweb-ping --url=http://localhost:8270/crabserver/preprod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" + failureThreshold: 3 + initialDelaySeconds: 120 + periodSeconds: 60 + timeoutSeconds: 60 + +readinessProbe: + exec: + command: + - /bin/sh + - -c + - | + cmsweb-ping --url=http://localhost:8270/crabserver/preprod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" + periodSeconds: 60 + timeoutSeconds: 60 + +logPipeline: + enabled: true + # logstashNamespace: logging diff --git a/helm/crabserver/values-prod.yaml b/helm/crabserver/values-prod.yaml index bc98e27a7..b7550aedb 100644 --- a/helm/crabserver/values-prod.yaml +++ b/helm/crabserver/values-prod.yaml @@ -1,10 +1,38 @@ environment: "prod" +replicaCount: 15 + image: - tag: "v3.250401-stable" + tag: "v3.251002-stable" canary: enabled: true replicaCount: 1 image: - tag: "v3.250401-stable" + tag: "v3.251002-stable" + +livenessProbe: + exec: + command: + - /bin/sh + - -c + - | + cmsweb-ping --url=http://localhost:8270/crabserver/prod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" + failureThreshold: 3 + initialDelaySeconds: 120 + periodSeconds: 60 + timeoutSeconds: 60 + +readinessProbe: + exec: + command: + - /bin/sh + - -c + - | + cmsweb-ping --url=http://localhost:8270/crabserver/prod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" + periodSeconds: 60 + timeoutSeconds: 60 + +logPipeline: + enabled: true + # logstashNamespace: logging diff --git a/helm/crabserver/values-test12.yaml b/helm/crabserver/values-test12.yaml new file mode 100644 index 000000000..6c22e7c02 --- /dev/null +++ b/helm/crabserver/values-test12.yaml @@ -0,0 +1,5 @@ +--- +environment: "test" + +image: + tag: "pypi-test12-290925-4" diff --git a/helm/crabserver/values-test14.yaml b/helm/crabserver/values-test14.yaml new file mode 100644 index 000000000..b0775b330 --- /dev/null +++ b/helm/crabserver/values-test14.yaml @@ -0,0 +1,5 @@ +--- +environment: "test" + +image: + tag: "pypi-test14-1759241248" diff --git a/helm/crabserver/values-testx.yaml b/helm/crabserver/values-test2.yaml similarity index 53% rename from helm/crabserver/values-testx.yaml rename to helm/crabserver/values-test2.yaml index 163272844..7648d7746 100644 --- a/helm/crabserver/values-testx.yaml +++ b/helm/crabserver/values-test2.yaml @@ -2,4 +2,4 @@ environment: "test" image: - tag: "v3.240809-stable" + tag: "v3.250929.1-stable" diff --git a/helm/crabserver/values-testbed.yaml b/helm/crabserver/values-testbed.yaml deleted file mode 100644 index 64a304e6f..000000000 --- a/helm/crabserver/values-testbed.yaml +++ /dev/null @@ -1,4 +0,0 @@ -environment: "preprod" - -image: - tag: "v3.240904-stable" diff --git a/helm/crabserver/values-testx-noprobe.yaml b/helm/crabserver/values-testx-noprobe.yaml deleted file mode 100644 index 6f16d2302..000000000 --- a/helm/crabserver/values-testx-noprobe.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -environment: "test" - -image: - tag: "v3.240809-stable" - -# disable liveness/readiness -# https://helm.sh/docs/chart_template_guide/values_files/#deleting-a-default-key -livenessProbePreProd: null -readinessProbePreProd: null -livenessProbeTest: null -readinessProbeTest: null -livenessProbe: null -readinessProbe: null diff --git a/helm/crabserver/values.yaml b/helm/crabserver/values.yaml index dc3378b4a..85d309b57 100644 --- a/helm/crabserver/values.yaml +++ b/helm/crabserver/values.yaml @@ -1,13 +1,12 @@ -# Default values for crabserver. -# This is a YAML-formatted file. +# Default values for crabserver.# This is a YAML-formatted file. # Declare variables to be passed into your templates. enabled: true -replicaCount: - prod: 15 - preprod: 1 - test: 1 +replicaCount: 1 + +crabserver: + configPy: null image: path: registry.cern.ch/cmscrab/crabserver @@ -27,6 +26,7 @@ image: canary: enabled: false + replicaCount: 1 image: path: registry.cern.ch/cmscrab/crabserver pullPolicy: IfNotPresent @@ -78,12 +78,6 @@ serviceMon: port: 18720 protocol: TCP -ingress: - enabled: false - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - resources: limits: cpu: 1500m @@ -93,50 +87,6 @@ resources: memory: 256Mi livenessProbe: - exec: - command: - - /bin/sh - - -c - - | - cmsweb-ping --url=http://localhost:8270/crabserver/prod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" - failureThreshold: 3 - initialDelaySeconds: 120 - periodSeconds: 60 - timeoutSeconds: 60 - -readinessProbe: - exec: - command: - - /bin/sh - - -c - - | - cmsweb-ping --url=http://localhost:8270/crabserver/prod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" - periodSeconds: 60 - timeoutSeconds: 60 - -livenessProbePreProd: - exec: - command: - - /bin/sh - - -c - - | - cmsweb-ping --url=http://localhost:8270/crabserver/preprod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" - failureThreshold: 3 - initialDelaySeconds: 120 - periodSeconds: 60 - timeoutSeconds: 60 - -readinessProbePreProd: - exec: - command: - - /bin/sh - - -c - - | - cmsweb-ping --url=http://localhost:8270/crabserver/preprod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" - periodSeconds: 60 - timeoutSeconds: 60 - -livenessProbeTest: exec: command: - /bin/sh @@ -148,7 +98,7 @@ livenessProbeTest: periodSeconds: 60 timeoutSeconds: 60 -readinessProbeTest: +readinessProbe: exec: command: - /bin/sh @@ -171,12 +121,12 @@ tolerations: [] affinity: {} - ciServiceAccount: enabled: false logPipeline: enabled: false + logstashNamespace: crab filebeat: name: filebeat path: docker.elastic.co/beats/filebeat:8.5.1