From 0e9f46f0e8ca85789abc9b6c373931d8b455befc Mon Sep 17 00:00:00 2001 From: Robert Grandl Date: Thu, 13 Jul 2023 15:15:02 -0700 Subject: [PATCH] add rollout support to weaver kube In the current implementation, with weaver kube you have to pause the world when you want to rollout a new version of the app. In this PR we enable rolling upgrades, a deployment strategy that is supported by Kubernetes. In the near future, we might want to provide integration with argo rollouts (or something else) to enable blue/green and canary deployments. Main changes: * for each component that exports a listener, we create a deployment with an unique name; this doesn't change across rollouts, so a rolling update will update the existing pods one by one * one a pod is updated to the new version, whenever it gets a request, it will only talk with other services within the same version * for each public listener, the name should be unique and not deployment specific * also, for jaeger and prometheus, we want unique instances that persist across multiple versions (and captures the metrics/traces for all the deployer versins of an app) * Prometheus used to scrape an address that was passed to the pods to export metrics on; this doesn't really work across versions; so we change the prometheus config to scrape all pods based on a label (the label is based on the app name, so it scrapes all pods across all versions of the app) Drawbacks of this approach (things that might not work): * what if a listener moves between components * what if a listener is dropped For now, in all our examples we have a simple listener exported by main. So we can restrict to be only one listener exported by main, and this might cover most of the scenarios. However, how to handle listeners across versions is something to think about. --- go.mod | 22 ++-- go.sum | 96 ++++++++------ internal/impl/babysitter.go | 41 +++--- internal/impl/kube.go | 248 +++++++++++++++++++++--------------- 4 files changed, 232 insertions(+), 175 deletions(-) diff --git a/go.mod b/go.mod index 40d3847..720f001 100644 --- a/go.mod +++ b/go.mod @@ -1,42 +1,40 @@ module github.com/ServiceWeaver/weaver-kube -go 1.20 +go 1.21 require ( github.com/ServiceWeaver/weaver v0.17.0 github.com/google/uuid v1.3.0 go.opentelemetry.io/otel/exporters/jaeger v1.16.0 go.opentelemetry.io/otel/sdk v1.16.0 - golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea - google.golang.org/protobuf v1.29.1 + golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 + google.golang.org/protobuf v1.31.0 k8s.io/api v0.26.2 k8s.io/apimachinery v0.27.2 sigs.k8s.io/yaml v1.3.0 ) require ( - github.com/BurntSushi/toml v1.2.0 // indirect - github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed // indirect + github.com/BurntSushi/toml v1.3.2 // indirect + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/google/cel-go v0.12.5 // indirect + github.com/google/cel-go v0.17.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect go.opentelemetry.io/otel v1.16.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.7.0 // indirect go.opentelemetry.io/otel/metric v1.16.0 // indirect go.opentelemetry.io/otel/trace v1.16.0 // indirect - golang.org/x/mod v0.9.0 // indirect golang.org/x/net v0.8.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/term v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/term v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 5e87483..fa13451 100644 --- a/go.sum +++ b/go.sum @@ -1,19 +1,20 @@ -github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= -github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/DataDog/hyperloglog v0.0.0-20220214164406-974598347557 h1:t+/ZxFkPEVJYvbKbk4suUhKFuCy3C8rg3tuP6Oj/Zyk= -github.com/ServiceWeaver/weaver v0.17.0 h1:5AwaK6z6Q3sR2cJWXydeku5myDVcesmK6S+AJlnPu/w= -github.com/ServiceWeaver/weaver v0.17.0/go.mod h1:zvziH3h+Blab1yppwDsmaZks2Zd5C3/YmSNs6LCODSY= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed h1:ue9pVfIcP+QMEjfgo/Ez4ZjNZfonGgR6NgjMaJMu1Cg= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/DataDog/hyperloglog v0.0.0-20220804205443-1806d9b66146 h1:S5WsRc58vIeuhvbz0V0FKs19nTbh5z23DCutLIXJkFA= +github.com/DataDog/hyperloglog v0.0.0-20220804205443-1806d9b66146/go.mod h1:hFPkswc42pKhRbeKDKXy05mRi7J1kJ2vMNbvd9erH0M= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -21,70 +22,75 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/google/cel-go v0.12.5 h1:DmzaiSgoaqGCjtpPQWl26/gND+yRpim56H1jCVev6d8= -github.com/google/cel-go v0.12.5/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= +github.com/google/cel-go v0.17.0 h1:o8fqHUcM+0g5prwg4pHZR+EMdef2RBumnEYefCXAyPQ= +github.com/google/cel-go v0.17.0/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+uqi7GyzaLa5MH7qlSLBZtRdiA= +github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/lightstep/varopt v1.3.0 h1:H7OhtEBhYyDhoMu+wJGl4mTqM9TrYYdThG+xLGU3fZQ= +github.com/lightstep/varopt v1.3.0/go.mod h1:3GP18zB7pfvbVUAnJ8xfvYjpwp0CF027QRD5FsfXau0= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 h1:pginetY7+onl4qN1vl0xW/V/v6OBZ0vVdH+esuJgvmM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0/go.mod h1:XiYsayHc36K3EByOO6nbAXnAWbrUxdjUROCEeeROOH8= go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= go.opentelemetry.io/otel/exporters/jaeger v1.16.0 h1:YhxxmXZ011C0aDZKoNw+juVWAmEfv/0W2XBOv9aHTaA= go.opentelemetry.io/otel/exporters/jaeger v1.16.0/go.mod h1:grYbBo/5afWlPpdPZYhyn78Bk04hnvxn2+hvxQhKIQM= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.7.0 h1:8hPcgCg0rUJiKE6VWahRvjgLUrNl7rW2hffUEPKXVEM= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.7.0/go.mod h1:K4GDXPY6TjUiwbOh+DkKaEdCF8y+lvMoM6SeAPyfCCM= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 h1:+XWJd3jf75RXJq29mxbuXhCXFDG3S3R4vBUeSI2P7tE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0/go.mod h1:hqgzBPTf4yONMFgdZvL/bK42R/iinTyVQtiWihs3SZc= go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= -go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU= go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= -go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea h1:vLCWI/yYrdEHyN2JzIzPO3aaQJHQdp89IZBA/+azVC4= -golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -94,26 +100,26 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= +golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -121,8 +127,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= -google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -131,8 +137,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.26.2 h1:dM3cinp3PGB6asOySalOZxEG4CZ0IAdJsrYZXE/ovGQ= k8s.io/api v0.26.2/go.mod h1:1kjMQsFE+QHPfskEcVNgL3+Hp88B80uj0QtSOlj8itU= k8s.io/apimachinery v0.27.2 h1:vBjGaKKieaIreI+oQwELalVG4d8f3YAMNpWLzDXkxeg= @@ -142,15 +148,25 @@ k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= +modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= -modernc.org/libc v1.22.3 h1:D/g6O5ftAfavceqlLOFwaZuA5KYafKwmr30A6iSqoyY= +modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= +modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= +modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/sqlite v1.21.0 h1:4aP4MdUf15i3R3M2mx6Q90WHKz3nZLoz96zlB6tNdow= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.24.0 h1:EsClRIWHGhLTCX44p+Ri/JLD+vFGo0QGjasg2/F9TlI= +modernc.org/sqlite v1.24.0/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg= +modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= diff --git a/internal/impl/babysitter.go b/internal/impl/babysitter.go index 2bffce9..8a6e977 100644 --- a/internal/impl/babysitter.go +++ b/internal/impl/babysitter.go @@ -28,8 +28,9 @@ import ( "github.com/ServiceWeaver/weaver/runtime/envelope" "github.com/ServiceWeaver/weaver/runtime/logging" "github.com/ServiceWeaver/weaver/runtime/metrics" - imetrics "github.com/ServiceWeaver/weaver/runtime/prometheus" + "github.com/ServiceWeaver/weaver/runtime/prometheus" "github.com/ServiceWeaver/weaver/runtime/protos" + "github.com/ServiceWeaver/weaver/runtime/traces" "github.com/google/uuid" "go.opentelemetry.io/otel/exporters/jaeger" "go.opentelemetry.io/otel/sdk/trace" @@ -43,10 +44,10 @@ const prometheusEndpoint = "/metrics" // babysitter starts and manages a weavelet inside the Pod. type babysitter struct { - ctx context.Context - cfg *ReplicaSetConfig - envelope *envelope.Envelope - traceSaver func(spans []trace.ReadOnlySpan) error + ctx context.Context + cfg *ReplicaSetConfig + envelope *envelope.Envelope + exportTraces func(spans *protos.TraceSpans) error // printer pretty prints log entries. printer *logging.PrettyPrinter @@ -81,7 +82,7 @@ func RunBabysitter(ctx context.Context) error { } // Create the trace exporter. - collector := name{cfg.Deployment.App.Name, jaegerAppName, cfg.Deployment.Id[:8]}.DNSLabel() + collector := name{cfg.Deployment.App.Name, jaegerAppName}.DNSLabel() endpoint := fmt.Sprintf("http://%s:%d/api/traces", collector, jaegerCollectorPort) traceExporter, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(endpoint))) @@ -90,17 +91,21 @@ func RunBabysitter(ctx context.Context) error { } defer traceExporter.Shutdown(ctx) //nolint:errcheck // response write error - traceSaver := func(spans []trace.ReadOnlySpan) error { - return traceExporter.ExportSpans(ctx, spans) + exportTraces := func(spans *protos.TraceSpans) error { + var spansToExport []trace.ReadOnlySpan + for _, span := range spans.Span { + spansToExport = append(spansToExport, &traces.ReadSpan{Span: span}) + } + return traceExporter.ExportSpans(ctx, spansToExport) } // Create the babysitter. b := &babysitter{ - ctx: ctx, - cfg: cfg, - envelope: e, - traceSaver: traceSaver, - printer: logging.NewPrettyPrinter(colors.Enabled()), + ctx: ctx, + cfg: cfg, + envelope: e, + exportTraces: exportTraces, + printer: logging.NewPrettyPrinter(colors.Enabled()), } // Inform the weavelet of the components it should host. @@ -122,7 +127,7 @@ func RunBabysitter(ctx context.Context) error { // Read the metrics. metrics := b.readMetrics() var b bytes.Buffer - imetrics.TranslateMetricsToPrometheusTextFormat(&b, metrics, r.Host, prometheusEndpoint) + prometheus.TranslateMetricsToPrometheusTextFormat(&b, metrics, r.Host, prometheusEndpoint) w.Write(b.Bytes()) //nolint:errcheck // response write error }) go func() { @@ -182,11 +187,11 @@ func (b *babysitter) HandleLogEntry(_ context.Context, entry *protos.LogEntry) e } // HandleTraceSpans implements the envelope.EnvelopeHandler interface. -func (b *babysitter) HandleTraceSpans(_ context.Context, spans []trace.ReadOnlySpan) error { - if b.traceSaver == nil { +func (b *babysitter) HandleTraceSpans(_ context.Context, spans *protos.TraceSpans) error { + if b.exportTraces == nil { return nil } - return b.traceSaver(spans) + return b.exportTraces(spans) } // GetSelfCertificate implements the envelope.EnvelopeHandler interface. @@ -200,7 +205,7 @@ func (b *babysitter) VerifyClientCertificate(context.Context, *protos.VerifyClie } // VerifyServerCertificate implements the envelope.EnvelopeHandler interface. -func (b *babysitter) VerifyServerCertificate(ctx context.Context, request *protos.VerifyServerCertificateRequest) (*protos.VerifyServerCertificateReply, error) { +func (b *babysitter) VerifyServerCertificate(context.Context, *protos.VerifyServerCertificateRequest) (*protos.VerifyServerCertificateReply, error) { panic("unused") } diff --git a/internal/impl/kube.go b/internal/impl/kube.go index fb5a2e6..e22097c 100644 --- a/internal/impl/kube.go +++ b/internal/impl/kube.go @@ -18,16 +18,15 @@ import ( "fmt" "os" "path/filepath" - "strings" "github.com/ServiceWeaver/weaver-kube/internal/proto" "github.com/ServiceWeaver/weaver/runtime/bin" "github.com/ServiceWeaver/weaver/runtime/protos" - _ "go.opentelemetry.io/otel/exporters/jaeger" "golang.org/x/exp/maps" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/apps/v1" v2 "k8s.io/api/autoscaling/v2" + _ "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -39,16 +38,6 @@ const ( // Name of the container that hosts the application binary. appContainerName = "serviceweaver" - // Key in a Kubernetes resource's label map that corresponds to the - // application name that the resource is associated with. Used when looking - // up resources that belong to a particular application. - appNameKey = "serviceweaver/app_name" - - // Key in Kubernetes resource's label map that corresponds to the - // application's deployment version. Used when looking up resources that - // belong to a particular application version. - deploymentIDKey = "serviceweaver/deployment_id" - // kubeConfigEnvKey is the name of the env variable that contains deployment // information for a babysitter deployed using kube. kubeConfigEnvKey = "SERVICEWEAVER_DEPLOYMENT_CONFIG" @@ -148,6 +137,35 @@ type KubeConfig struct { // GenerateKubeDeployment generates the kubernetes deployment and service // information for a given app deployment. +// +// Note that for each application, we generate the following Kubernetes topology: +// +// - for each replica set that has at least a listener, we generate a deployment +// whose name is persistent across new app versions rollouts. Note that in +// general this should only be the replica set that contains the main component. +// We do this because by default we rely on RollingUpdate as a deployment +// strategy to rollout new versions of the app. RollingUpdate will update the +// existing pods for the replica set with the new version one by one, hence +// the new application version is being deployed. +// For example, let's assume that we have an app v1 with 2 replica sets main +// and foo. main is the replica set that contains the public listener, and it +// has 2 replicas. Next, we deploy the version v2 of the app. v2 will be +// rolled out as follows: +// +// [main v1] [main v1] [main v1] [main v2] [main v2] [main v2] +// | | | | | | +// v | => v v => | v +// [foo v1] <---| [foo v1] [foo v2] |-------> [foo v2] +// +// - for all the replica sets, we create a per app version service so components +// within an app version can communicate with each other. +// +// - for all the public listeners we create a load balancer with a unique name +// that is persistent across new app version rollouts. +// +// - if Prometheus/Jaeger are enabled, we deploy corresponding services using +// unique names as well s.t., we don't rollout new instances of Prometheus/Jaeger, +// when we rollout new versions of the app. func GenerateKubeDeployment(image string, dep *protos.Deployment, cfg *KubeConfig) error { fmt.Fprintf(os.Stderr, greenText(), "\nGenerating kube deployment info ...") @@ -173,7 +191,7 @@ func GenerateKubeDeployment(image string, dep *protos.Deployment, cfg *KubeConfi generated = append(generated, content...) // Generate the Prometheus deployment info. - content, err = generatePrometheusDeployment(maps.Values(replicaSets), dep) + content, err = generatePrometheusDeployment(dep) if err != nil { return fmt.Errorf("unable to create kube deployment for the Prometheus service: %w", err) } @@ -268,8 +286,14 @@ func generateAppDeployment(replicaSets map[string]*replicaSetInfo, image string, // generateJaegerDeployment generates the Jaeger kubernetes deployment and service // information for a given app deployment. +// +// Note that we run a single instance of Jaeger. This is because we are using +// a Jaeger image that combines three Jaeger components, agent, collector, and +// query service/UI in a single image. +// TODO(rgrandl): If the trace volume can't be handled by a single instance, we +// should scale these components independently, and use different image(s). func generateJaegerDeployment(dep *protos.Deployment) ([]byte, error) { - name := name{dep.App.Name, jaegerAppName, dep.Id[:8]}.DNSLabel() + jname := name{dep.App.Name, jaegerAppName}.DNSLabel() // Generate the Jaeger deployment. d := &appsv1.Deployment{ @@ -277,35 +301,34 @@ func generateJaegerDeployment(dep *protos.Deployment) ([]byte, error) { APIVersion: "apps/v1", Kind: "Deployment", }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{deploymentIDKey: dep.Id}, - }, + ObjectMeta: metav1.ObjectMeta{Name: jname}, Spec: appsv1.DeploymentSpec{ Replicas: ptrOf(int32(1)), Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "app": name, - "dep_id": dep.Id, + "jaeger": jname, }, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "app": name, - "dep_id": dep.Id, + "jaeger": jname, }, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: name, + Name: jname, Image: fmt.Sprintf("%s:latest", jaegerImageName), ImagePullPolicy: corev1.PullIfNotPresent, }, }, }, }, + Strategy: v1.DeploymentStrategy{ + Type: "RollingUpdate", + RollingUpdate: &v1.RollingUpdateDeployment{}, + }, }, } content, err := yaml.Marshal(d) @@ -324,15 +347,9 @@ func generateJaegerDeployment(dep *protos.Deployment) ([]byte, error) { APIVersion: "v1", Kind: "Service", }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{deploymentIDKey: dep.Id}, - }, + ObjectMeta: metav1.ObjectMeta{Name: jname}, Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "app": name, - "dep_id": dep.Id, - }, + Selector: map[string]string{"jaeger": jname}, Ports: []corev1.ServicePort{ { Name: "ui-port", @@ -357,6 +374,7 @@ func generateJaegerDeployment(dep *protos.Deployment) ([]byte, error) { generated = append(generated, content...) generated = append(generated, []byte("\n---\n")...) fmt.Fprintf(os.Stderr, "Generated Jaeger service\n") + return generated, nil } @@ -364,35 +382,37 @@ func generateJaegerDeployment(dep *protos.Deployment) ([]byte, error) { // a Prometheus service for a given app deployment. // // TODO(rgrandl): check if we can simplify the config map, and the deployment info. -func generatePrometheusDeployment(rs []*replicaSetInfo, dep *protos.Deployment) ([]byte, error) { - cname := name{dep.App.Name, "prometheus", "config", dep.Id[:8]}.DNSLabel() - pname := name{dep.App.Name, "prometheus", dep.Id[:8]}.DNSLabel() - - // Build the list of monitoring targets that should be scraped by Prometheus. - var targetsStr []string - for _, r := range rs { - tname := fmt.Sprintf("\n\"%s:%d\"", name{dep.App.Name, r.name, dep.Id[:8]}.DNSLabel(), metricsPort) - targetsStr = append(targetsStr, tname) - } - // Build the config map that holds the prometheus configuration file. +// +// TODO(rgrandl): We run a single instance of Prometheus for now. We might want +// to scale it up if it becomes a bottleneck. +func generatePrometheusDeployment(dep *protos.Deployment) ([]byte, error) { + cname := name{dep.App.Name, "prometheus", "config"}.DNSLabel() + pname := name{dep.App.Name, "prometheus"}.DNSLabel() + + // Build the config map that holds the prometheus configuration file. In the + // config we specify how to scrape the app pods for the metrics. config := fmt.Sprintf(` global: scrape_interval: 15s scrape_configs: - job_name: "%s" - metrics_path: /metrics - static_configs: - - targets: [%v] -`, pname, strings.Join(targetsStr, ",")) - + metrics_path: %s + kubernetes_sd_configs: + - role: pod + scheme: http + relabel_configs: + - source_labels: [__meta_kubernetes_pod_label_metrics] + regex: "%s" + action: keep +`, pname, prometheusEndpoint, dep.App.Name) + + // Create a config map to store the prometheus config. cm := corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ Kind: "ConfigMap", APIVersion: "v1", }, - ObjectMeta: metav1.ObjectMeta{ - Name: cname, - }, + ObjectMeta: metav1.ObjectMeta{Name: cname}, Data: map[string]string{ "prometheus.yaml": config, }, @@ -413,17 +433,15 @@ scrape_configs: APIVersion: "apps/v1", Kind: "Deployment", }, - ObjectMeta: metav1.ObjectMeta{ - Name: pname, - }, + ObjectMeta: metav1.ObjectMeta{Name: pname}, Spec: appsv1.DeploymentSpec{ Replicas: ptrOf(int32(1)), Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": pname}, + MatchLabels: map[string]string{"prometheus": pname}, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": pname}, + Labels: map[string]string{"prometheus": pname}, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -472,6 +490,10 @@ scrape_configs: }, }, }, + Strategy: v1.DeploymentStrategy{ + Type: "RollingUpdate", + RollingUpdate: &v1.RollingUpdateDeployment{}, + }, }, } content, err = yaml.Marshal(d) @@ -491,7 +513,7 @@ scrape_configs: }, ObjectMeta: metav1.ObjectMeta{Name: pname}, Spec: corev1.ServiceSpec{ - Selector: map[string]string{"app": pname}, + Selector: map[string]string{"prometheus": pname}, Ports: []corev1.ServicePort{ { Port: servicePort, @@ -514,7 +536,24 @@ scrape_configs: // buildDeployment generates a kubernetes deployment for a replica set. func buildDeployment(rs *replicaSetInfo, dep *protos.Deployment, image string) (*v1.Deployment, error) { - name := name{dep.App.Name, rs.name, dep.Id[:8]}.DNSLabel() + // Unique name that persists across app versions. + globalName := name{dep.App.Name, rs.name}.DNSLabel() + + // Per deployment name that is app version specific. + depName := name{dep.App.Name, rs.name, dep.Id[:8]}.DNSLabel() + + var oname string + matchLabels := map[string]string{} + podLabels := map[string]string{"depName": depName, "metrics": dep.App.Name} + if hasListenersFn(rs) { + oname = globalName + matchLabels["globalName"] = globalName + podLabels["globalName"] = globalName + } else { + oname = depName + matchLabels["depName"] = depName + } + container, err := buildContainer(image, rs, dep) if err != nil { return nil, err @@ -524,57 +563,43 @@ func buildDeployment(rs *replicaSetInfo, dep *protos.Deployment, image string) ( APIVersion: "apps/v1", Kind: "Deployment", }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{ - appNameKey: dep.App.Name, - deploymentIDKey: dep.Id, - }, - }, + ObjectMeta: metav1.ObjectMeta{Name: oname}, Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": name, - "app_name": dep.App.Name, - "dep_id": dep.Id, - }, + MatchLabels: matchLabels, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": name, - "app_name": dep.App.Name, - "dep_id": dep.Id, - }, + Labels: podLabels, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{container}, }, }, + Strategy: v1.DeploymentStrategy{ + Type: "RollingUpdate", + RollingUpdate: &v1.RollingUpdateDeployment{}, + }, + // Number of old ReplicaSets to retain to allow rollback. + RevisionHistoryLimit: ptrOf(int32(1)), + MinReadySeconds: int32(5), }, }, nil } // buildService generates a kubernetes service for a replica set. func buildService(rs *replicaSetInfo, dep *protos.Deployment) (*corev1.Service, error) { - name := name{dep.App.Name, rs.name, dep.Id[:8]}.DNSLabel() + // Per deployment name that is app version specific. + depName := name{dep.App.Name, rs.name, dep.Id[:8]}.DNSLabel() return &corev1.Service{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "Service", }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{ - appNameKey: dep.App.Name, - deploymentIDKey: dep.Id, - }, - }, + ObjectMeta: metav1.ObjectMeta{Name: depName}, Spec: corev1.ServiceSpec{ Selector: map[string]string{ - "app": name, - "app_name": dep.App.Name, - "dep_id": dep.Id, + "depName": depName, }, Ports: []corev1.ServicePort{ { @@ -600,34 +625,28 @@ func buildService(rs *replicaSetInfo, dep *protos.Deployment) (*corev1.Service, // it has to be reachable from the outside; for internal listeners, we generate // a ClusterIP service, reachable only from internal Service Weaver services. func buildListenerService(rs *replicaSetInfo, lis *ReplicaSetConfig_Listener, dep *protos.Deployment) (*corev1.Service, error) { - appName := name{dep.App.Name, rs.name, dep.Id[:8]}.DNSLabel() - lisName := name{dep.App.Name, "lis", rs.name, dep.Id[:8]}.DNSLabel() + // Unique name that persists across app versions. + globalLisName := name{dep.App.Name, "lis", rs.name}.DNSLabel() + var serviceType string + labels := map[string]string{} if lis.IsPublic { serviceType = "LoadBalancer" + labels["globalName"] = name{dep.App.Name, rs.name}.DNSLabel() } else { serviceType = "ClusterIP" + labels["depName"] = name{dep.App.Name, rs.name, dep.Id[:8]}.DNSLabel() } + return &corev1.Service{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "Service", }, - - ObjectMeta: metav1.ObjectMeta{ - Name: lisName, - Labels: map[string]string{ - appNameKey: dep.App.Name, - deploymentIDKey: dep.Id, - }, - }, + ObjectMeta: metav1.ObjectMeta{Name: globalLisName}, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceType(serviceType), - Selector: map[string]string{ - "app": appName, - "app_name": dep.App.Name, - "dep_id": dep.Id, - }, + Type: corev1.ServiceType(serviceType), + Selector: labels, Ports: []corev1.ServicePort{ { Port: servicePort, @@ -641,16 +660,21 @@ func buildListenerService(rs *replicaSetInfo, lis *ReplicaSetConfig_Listener, de // buildAutoscaler generates a kubernetes horizontal pod autoscaler for a replica set. func buildAutoscaler(rs *replicaSetInfo, dep *protos.Deployment) (*v2.HorizontalPodAutoscaler, error) { + // Per deployment name that is app version specific. aname := name{dep.App.Name, "hpa", rs.name, dep.Id[:8]}.DNSLabel() - depName := name{dep.App.Name, rs.name, dep.Id[:8]}.DNSLabel() + + var depName string + if hasListenersFn(rs) { + depName = name{dep.App.Name, rs.name}.DNSLabel() + } else { + depName = name{dep.App.Name, rs.name, dep.Id[:8]}.DNSLabel() + } return &v2.HorizontalPodAutoscaler{ TypeMeta: metav1.TypeMeta{ APIVersion: "autoscaling/v2", Kind: "HorizontalPodAutoscaler", }, - ObjectMeta: metav1.ObjectMeta{ - Name: aname, - }, + ObjectMeta: metav1.ObjectMeta{Name: aname}, Spec: v2.HorizontalPodAutoscalerSpec{ ScaleTargetRef: v2.CrossVersionObjectReference{ APIVersion: "apps/v1", @@ -716,6 +740,10 @@ func buildContainer(dockerImage string, rs *replicaSetInfo, dep *protos.Deployme // should be only for a short period of time. }, + // Expose the metrics port from the container, so it can be discoverable for + // scraping by Prometheus. + Ports: []corev1.ContainerPort{{ContainerPort: metricsPort}}, + // Enabling TTY and Stdin allows the user to run a shell inside the container, // for debugging. TTY: true, @@ -808,3 +836,13 @@ func getComponents(dep *protos.Deployment, cfg *KubeConfig) (map[string]*Replica } return components, nil } + +// hasListenersFn returns whether a given replica set exports any listeners. +func hasListenersFn(rs *replicaSetInfo) bool { + for _, listeners := range rs.components { + if listeners.Listeners != nil { + return true + } + } + return false +}