From d8f33d893c219fa0a7f4ddc0098269d2af4a5692 Mon Sep 17 00:00:00 2001 From: Adnan Gulegulzar Date: Thu, 6 Jun 2024 17:54:14 -0400 Subject: [PATCH 1/3] initialize plugin --- .gitignore | 3 +- .goreleaser.yml | 54 +++++ Makefile | 16 ++ README.md | 10 +- go.mod | 44 ++++ go.sum | 201 ++++++++++++++++++ main.go | 10 + plugin/README.md | 15 ++ plugin/gcp/README.md | 36 ++++ plugin/gcp/compute.go | 123 +++++++++++ plugin/gcp/compute_test.go | 61 ++++++ plugin/gcp/config.go | 37 ++++ plugin/preferences/default.go | 5 + .../compute_instance/compute_instance.go | 52 +++++ .../compute_instance/compute_instance_item.go | 43 ++++ .../job_compute_instance_list.go | 62 ++++++ plugin/processor/interface.go | 7 + plugin/service.go | 102 +++++++++ plugin/version/def.go | 3 + utils/concurrent_map.go | 35 +++ 20 files changed, 913 insertions(+), 6 deletions(-) create mode 100644 .goreleaser.yml create mode 100644 Makefile create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 plugin/README.md create mode 100644 plugin/gcp/README.md create mode 100644 plugin/gcp/compute.go create mode 100644 plugin/gcp/compute_test.go create mode 100644 plugin/gcp/config.go create mode 100644 plugin/preferences/default.go create mode 100644 plugin/processor/compute_instance/compute_instance.go create mode 100644 plugin/processor/compute_instance/compute_instance_item.go create mode 100644 plugin/processor/compute_instance/job_compute_instance_list.go create mode 100644 plugin/processor/interface.go create mode 100644 plugin/service.go create mode 100644 plugin/version/def.go create mode 100644 utils/concurrent_map.go diff --git a/.gitignore b/.gitignore index 423dc8d..0fea7c1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea .kaytu -cli \ No newline at end of file +cli +.vscode \ No newline at end of file diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..15d246b --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,54 @@ +--- +project_name: kaytu + +release: + github: + owner: "{{ .Env.REPOSITORY_OWNER }}" + name: "{{ .Env.REPOSITORY_NAME }}" + prerelease: auto + make_latest: "{{ .Env.GORELEASER_MAKE_LATEST}}" + +checksum: {} + +builds: + - id: linux + binary: kaytu + ldflags: + - -s -w -X github.com/{{ .Env.REPOSITORY_OWNER }}/{{ .Env.REPOSITORY_NAME }}/pkg/version.VERSION={{ .Version }} + goos: + - linux + goarch: + - amd64 + - arm64 + main: ./main.go + - id: darwin + binary: kaytu + ldflags: + - -s -w -X github.com/{{ .Env.REPOSITORY_OWNER }}/{{ .Env.REPOSITORY_NAME }}/pkg/version.VERSION={{ .Version }} + goos: + - darwin + goarch: + - amd64 + - arm64 + main: ./main.go + - id: windows + binary: kaytu + ldflags: + - -s -w -X github.com/{{ .Env.REPOSITORY_OWNER }}/{{ .Env.REPOSITORY_NAME }}/pkg/version.VERSION={{ .Version }} + goos: + - windows + goarch: + - amd64 + main: ./main.go +archives: + - id: binary + format: binary + - id: windows + format: zip + builds: [windows] + - id: linux + format: tar.gz + builds: [linux] + - id: darwin + format: tar.gz + builds: [darwin] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0817c00 --- /dev/null +++ b/Makefile @@ -0,0 +1,16 @@ +# run plugin in debug mode +# (Assuming Kaytu CLI is running in debug mode AND Kaytu CLI has plugin-gcp installed) +debug: + go run main.go --server localhost:30422 + +# removes the dist folder, which is generated by goreleaser build(make goreleaser) +clean: + rm -rf dist + +# cleans the dist directory and generates build using the goreleaser's configuration for current GOOS and GOARCH +goreleaser: clean + REPOSITORY_NAME="kaytu" REPOSITORY_OWNER="kaytu-io" goreleaser build --snapshot --single-target + +# test components of plugin/gcp +testgcp: + go test -v ./plugin/gcp \ No newline at end of file diff --git a/README.md b/README.md index 66e088a..c30260a 100644 --- a/README.md +++ b/README.md @@ -36,16 +36,16 @@ Download Windows (Linux, and MacOS) binary from [releases](https://github.com/ka ### 2. Login to AWS CLI -Kaytu works with your existing AWS CLI profile (read-only access required) to gather metrics. +Kaytu works with your existing Google Cloud CLI(gcloud) configuration (read-only access required) to gather metrics. -To confirm your AWS CLI login is working correctly: +To confirm your gcloud login is configured correctly: ``` -aws sts get-caller-identity +gcloud config list ``` -[Click here to see how to log in to AWS CLI.](https://docs.aws.amazon.com/signin/latest/userguide/command-line-sign-in.html) +[Click here to see how to configure Gcloud.](https://cloud.google.com/sdk/docs/initializing) -We respect your privacy. Our open-source code guarantees that we never collect sensitive information like AWS credentials, IPs, tags, etc. +We respect your privacy. Our open-source code guarantees that we never collect sensitive information like GCP credentials, IPs, tags, etc. ### 3. Run Kaytu CLI diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2e77d37 --- /dev/null +++ b/go.mod @@ -0,0 +1,44 @@ +module github.com/kaytu-io/plugin-gcp + +go 1.21.5 + +require ( + cloud.google.com/go/compute v1.24.0 + cloud.google.com/go/monitoring v1.18.0 + github.com/kaytu-io/kaytu v0.8.8 + golang.org/x/oauth2 v0.17.0 + google.golang.org/api v0.162.0 +) + +require ( + cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/cobra v1.8.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect + go.opentelemetry.io/otel v1.22.0 // indirect + go.opentelemetry.io/otel/metric v1.22.0 // indirect + go.opentelemetry.io/otel/trace v1.22.0 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect + google.golang.org/grpc v1.63.2 // indirect + google.golang.org/protobuf v1.34.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..1bd39a2 --- /dev/null +++ b/go.sum @@ -0,0 +1,201 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= +cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= +cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= +cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/monitoring v1.18.0 h1:NfkDLQDG2UR3WYZVQE8kwSbUIEyIqJUPl+aOQdFH1T4= +cloud.google.com/go/monitoring v1.18.0/go.mod h1:c92vVBCeq/OB4Ioyo+NbN2U7tlg5ZH41PZcdvfc+Lcg= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kaytu-io/kaytu v0.8.8 h1:BiqMr/CEskRuEBqCfANjCPki/SwZGW9/yXuLUyNFb/E= +github.com/kaytu-io/kaytu v0.8.8/go.mod h1:1Whjajpn9DSsEKbxX9BoWvQoQeOK0CZVEVxNB6RCQVc= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 h1:UNQQKPfTDe1J81ViolILjTKPr9WetKW6uei2hFgJmFs= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= +go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= +go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= +go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= +go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= +go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ= +golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.162.0 h1:Vhs54HkaEpkMBdgGdOT2P6F0csGG/vxDS0hWHJzmmps= +google.golang.org/api v0.162.0/go.mod h1:6SulDkfoBIg4NFmCuZ39XeeAgSHCPecfSUuDyYlAHs0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= +google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= +google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= +google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go new file mode 100644 index 0000000..8f092be --- /dev/null +++ b/main.go @@ -0,0 +1,10 @@ +package main + +import ( + "github.com/kaytu-io/kaytu/pkg/plugin/sdk" + "github.com/kaytu-io/plugin-gcp/plugin" +) + +func main() { + sdk.New(plugin.NewPlugin(), 4).Execute() +} diff --git a/plugin/README.md b/plugin/README.md new file mode 100644 index 0000000..5cfb789 --- /dev/null +++ b/plugin/README.md @@ -0,0 +1,15 @@ +# Plugin + +The plugin package implements the [`plugin sdk`](https://github.com/kaytu-io/kaytu/tree/main/pkg/plugin/sdk) of the kaytu repository. + +# Components + +- service.go: implements [processor interface](https://github.com/kaytu-io/kaytu/blob/main/pkg/plugin/sdk/processor.go) + +- [gcp](gcp/README.md): package with Google Cloud Platform components + +- preferences: package with default preferences for items in processor + +- [processor](processor/README.md): contains the processors for GCP plugin + +- versions: default versions \ No newline at end of file diff --git a/plugin/gcp/README.md b/plugin/gcp/README.md new file mode 100644 index 0000000..18403e9 --- /dev/null +++ b/plugin/gcp/README.md @@ -0,0 +1,36 @@ +# GCP package for wrapping GCP api + +## About each component + +- GCP + - Base configuration for all Google Cloud Clients + - Embedded by other clients e.g. Compute Instance types + +- Compute + - Controls Compute client for GCP + - Gets list of Instances + +- Metrics + - Controls Metrics client for GCP + - + + + +TODO: + + - Metrics Client + - Close metrics client + + - Metrics to collect: + - "CPUUtilization", + + - "NetworkIn", + - "NetworkOut", + + - "mem_used_percent", + + - "VolumeReadBytes", + - "VolumeWriteBytes", + + - "VolumeReadOps", + - "VolumeWriteOps", \ No newline at end of file diff --git a/plugin/gcp/compute.go b/plugin/gcp/compute.go new file mode 100644 index 0000000..9f9e9ba --- /dev/null +++ b/plugin/gcp/compute.go @@ -0,0 +1,123 @@ +// Google Cloud Compute Instances + +package gcp + +import ( + "context" + "log" + + compute "cloud.google.com/go/compute/apiv1" + "cloud.google.com/go/compute/apiv1/computepb" + "google.golang.org/api/iterator" + "google.golang.org/api/option" +) + +type Compute struct { + client *compute.InstancesClient + GCP +} + +func NewCompute(scopes []string) *Compute { + return &Compute{ + GCP: NewGCP(scopes), + } +} + +func (c *Compute) InitializeClient(ctx context.Context) error { + + c.GCP.GetCredentials(ctx) + + // log.Println(string(c.GCP.credentials.JSON)) + // log.Println(c.GCP.ProjectID) + + instancesClient, err := compute.NewInstancesRESTClient( + ctx, + option.WithCredentials(c.GCP.credentials), + ) + if err != nil { + return err + } + + // log.Println(instancesClient) + + c.client = instancesClient + + return nil +} + +func (c *Compute) CloseClient() error { + err := c.client.Close() + if err != nil { + return err + } + return nil +} + +func (c *Compute) ListAllInstances() error { + + req := &computepb.AggregatedListInstancesRequest{ + Project: c.ProjectID, + } + + it := c.client.AggregatedList(context.Background(), req) + + log.Println("instances found: ") + + for { + pair, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return err + } + + instances := pair.Value.Instances + + if len(instances) > 0 { + // log.Printf("%s\n", pair.Key) + for _, instance := range instances { + log.Printf("%s", instance.GetName()) + } + } + } + return nil +} + +func (c *Compute) GetAllInstances() ([]*computepb.Instance, error) { + + var allInstances []*computepb.Instance + + req := &computepb.AggregatedListInstancesRequest{ + Project: c.ProjectID, + } + + it := c.client.AggregatedList(context.Background(), req) + + log.Println("instances found: ") + + for { + pair, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, err + } + + instances := pair.Value.Instances + + // allInstances = append(allInstances, *instances) + + if len(instances) > 0 { + allInstances = append(allInstances, instances...) + + // log.Printf("%s\n", pair.Key) = append(allInstances, instances...) + // for _, instance := range instances { + // log.Printf("%s", instance.GetName()) + // allInstances = append(allInstances, *instance) + // } + } + } + return allInstances, nil +} diff --git a/plugin/gcp/compute_test.go b/plugin/gcp/compute_test.go new file mode 100644 index 0000000..36f5fc8 --- /dev/null +++ b/plugin/gcp/compute_test.go @@ -0,0 +1,61 @@ +package gcp + +import ( + "context" + "log" + "testing" +) + +func TestListAllInstances(t *testing.T) { + log.Printf("running %s", t.Name()) + compute := NewCompute( + []string{ + "https://www.googleapis.com/auth/compute.readonly", + }, + ) + err := compute.InitializeClient(context.Background()) + if err != nil { + t.Errorf("[%s]: %s", t.Name(), err.Error()) + } + + log.Printf("[%s]: %s", t.Name(), compute.ProjectID) + + err = compute.ListAllInstances() + if err != nil { + compute.CloseClient() + t.Errorf("[%s]: %s", t.Name(), err.Error()) + } + compute.CloseClient() + +} + +func TestGetAllInstances(t *testing.T) { + + log.Printf("running %s", t.Name()) + compute := NewCompute( + []string{ + "https://www.googleapis.com/auth/compute.readonly", + }, + ) + err := compute.InitializeClient(context.Background()) + if err != nil { + t.Errorf("[%s]: %s", t.Name(), err.Error()) + } + + log.Printf("[%s]: %s", t.Name(), compute.ProjectID) + + instances, err := compute.GetAllInstances() + if err != nil { + compute.CloseClient() + t.Errorf("[%s]: %s", t.Name(), err.Error()) + } + + for _, instance := range instances { + log.Println(instance.GetMachineType()) + } + + // log.Println(instances) + + compute.CloseClient() + +} diff --git a/plugin/gcp/config.go b/plugin/gcp/config.go new file mode 100644 index 0000000..144723f --- /dev/null +++ b/plugin/gcp/config.go @@ -0,0 +1,37 @@ +package gcp + +import ( + "context" + "encoding/json" + + "golang.org/x/oauth2/google" +) + +type GCP struct { + credentials *google.Credentials + ProjectID string `json:"quota_project_id"` + Scopes []string +} + +func NewGCP(scopes []string) GCP { + return GCP{ + ProjectID: "", + Scopes: scopes, + } +} + +func (g *GCP) GetCredentials(ctx context.Context) error { + credentials, err := google.FindDefaultCredentials( + ctx, + g.Scopes..., + ) + if err != nil { + return err + } + + json.Unmarshal(credentials.JSON, g) //this will store project id from credentials + + g.credentials = credentials + // log.Println(g.ProjectID) + return nil +} diff --git a/plugin/preferences/default.go b/plugin/preferences/default.go new file mode 100644 index 0000000..04568bb --- /dev/null +++ b/plugin/preferences/default.go @@ -0,0 +1,5 @@ +package preferences + +import "github.com/kaytu-io/kaytu/pkg/plugin/proto/src/golang" + +var DefaultComputeEnginePreferences = []*golang.PreferenceItem{} diff --git a/plugin/processor/compute_instance/compute_instance.go b/plugin/processor/compute_instance/compute_instance.go new file mode 100644 index 0000000..d694704 --- /dev/null +++ b/plugin/processor/compute_instance/compute_instance.go @@ -0,0 +1,52 @@ +package compute_instance + +import ( + "log" + + "github.com/kaytu-io/kaytu/pkg/plugin/proto/src/golang" + "github.com/kaytu-io/kaytu/pkg/plugin/sdk" + "github.com/kaytu-io/plugin-gcp/plugin/gcp" + util "github.com/kaytu-io/plugin-gcp/utils" +) + +type ComputeInstanceProcessor struct { + provider *gcp.Compute + // metricProvider *gcp.CloudMonitoring + identification map[string]string + items util.ConcurrentMap[string, ComputeInstanceItem] + publishOptimizationItem func(item *golang.OptimizationItem) + kaytuAcccessToken string + jobQueue *sdk.JobQueue +} + +func NewComputeInstanceProcessor( + prv *gcp.Compute, + // metric *gcp.CloudMonitoring, + publishOptimizationItem func(item *golang.OptimizationItem), + kaytuAcccessToken string, + jobQueue *sdk.JobQueue, +) *ComputeInstanceProcessor { + r := &ComputeInstanceProcessor{ + provider: prv, + // metricProvider: metric, + // identification: identification, + // items: util.NewMap[string, EC2InstanceItem](), + publishOptimizationItem: publishOptimizationItem, + kaytuAcccessToken: kaytuAcccessToken, + jobQueue: jobQueue, + // configuration: configurations, + // lazyloadCounter: lazyloadCounter, + } + log.Println("creating processor") + + jobQueue.Push(NewListComputeInstancesJob(r)) + return r +} + +func (m *ComputeInstanceProcessor) ReEvaluate(id string, items []*golang.PreferenceItem) { + log.Println("Reevaluate unimplemented") + // v, _ := m.items.Get(id) + // v.Preferences = items + // m.items.Set(id, v) + // m.jobQueue.Push(NewOptimizeEC2InstanceJob(m, v)) +} diff --git a/plugin/processor/compute_instance/compute_instance_item.go b/plugin/processor/compute_instance/compute_instance_item.go new file mode 100644 index 0000000..e929f6d --- /dev/null +++ b/plugin/processor/compute_instance/compute_instance_item.go @@ -0,0 +1,43 @@ +package compute_instance + +import ( + "github.com/kaytu-io/kaytu/pkg/plugin/proto/src/golang" +) + +type ComputeInstanceItem struct { + Name string + Id string + MachineType string + Region string + OptimizationLoading bool + Preferences []*golang.PreferenceItem + Skipped bool + LazyLoadingEnabled bool + SkipReason string + // Wastage kaytu.EC2InstanceWastageResponse +} + +func (i ComputeInstanceItem) ToOptimizationItem() *golang.OptimizationItem { + oi := &golang.OptimizationItem{ + Id: i.Id, + Name: i.Name, + ResourceType: i.MachineType, + Region: i.Region, + Devices: nil, + Preferences: i.Preferences, + Description: "description placeholder", + Loading: i.OptimizationLoading, + Skipped: i.Skipped, + SkipReason: i.SkipReason, + LazyLoadingEnabled: i.LazyLoadingEnabled, + } + + // if i.Instance.PlatformDetails != nil { + // oi.Platform = *i.Instance.PlatformDetails + // } + if oi.Name == "" { + oi.Name = string(i.Name) + } + + return oi +} diff --git a/plugin/processor/compute_instance/job_compute_instance_list.go b/plugin/processor/compute_instance/job_compute_instance_list.go new file mode 100644 index 0000000..2331072 --- /dev/null +++ b/plugin/processor/compute_instance/job_compute_instance_list.go @@ -0,0 +1,62 @@ +package compute_instance + +import ( + "log" + "strconv" + + "github.com/kaytu-io/plugin-gcp/plugin/preferences" +) + +type ListComputeInstancesJob struct { + processor *ComputeInstanceProcessor +} + +func NewListComputeInstancesJob(processor *ComputeInstanceProcessor) *ListComputeInstancesJob { + return &ListComputeInstancesJob{ + processor: processor, + } +} + +func (job *ListComputeInstancesJob) Id() string { + return "list_compute_instances" +} + +func (job *ListComputeInstancesJob) Description() string { + return "List all compute instances in current project" + +} + +func (job *ListComputeInstancesJob) Run() error { + + log.Println("Running list compute instance job") + + instances, err := job.processor.provider.GetAllInstances() + if err != nil { + return err + } + + for _, instance := range instances { + oi := ComputeInstanceItem{ + Name: *instance.Name, + Id: strconv.FormatUint(instance.GetId(), 10), + MachineType: instance.GetMachineType(), + Region: instance.GetZone(), + OptimizationLoading: false, + Preferences: preferences.DefaultComputeEnginePreferences, + Skipped: false, + LazyLoadingEnabled: false, + SkipReason: "NA", + // Instance: *instance, + // Region: j.region, + // OptimizationLoading: true, + // LazyLoadingEnabled: false, + // Preferences: preferences2.DefaultEC2Preferences, + } + + job.processor.items.Set(oi.Id, oi) + job.processor.publishOptimizationItem(oi.ToOptimizationItem()) + } + + return job.processor.provider.ListAllInstances() + +} diff --git a/plugin/processor/interface.go b/plugin/processor/interface.go new file mode 100644 index 0000000..9b0a71e --- /dev/null +++ b/plugin/processor/interface.go @@ -0,0 +1,7 @@ +package processor + +import "github.com/kaytu-io/kaytu/pkg/plugin/proto/src/golang" + +type PluginProcessor interface { + ReEvaluate(id string, items []*golang.PreferenceItem) +} diff --git a/plugin/service.go b/plugin/service.go new file mode 100644 index 0000000..bb8754d --- /dev/null +++ b/plugin/service.go @@ -0,0 +1,102 @@ +package plugin + +import ( + "fmt" + "log" + + "github.com/kaytu-io/kaytu/pkg/plugin/proto/src/golang" + "github.com/kaytu-io/kaytu/pkg/plugin/sdk" + "github.com/kaytu-io/plugin-gcp/plugin/gcp" + "github.com/kaytu-io/plugin-gcp/plugin/preferences" + "github.com/kaytu-io/plugin-gcp/plugin/processor" + "github.com/kaytu-io/plugin-gcp/plugin/processor/compute_instance" + "github.com/kaytu-io/plugin-gcp/plugin/version" +) + +type GCPPlugin struct { + stream golang.Plugin_RegisterClient + processor processor.PluginProcessor +} + +func NewPlugin() *GCPPlugin { + return &GCPPlugin{} +} + +func (p *GCPPlugin) GetConfig() golang.RegisterConfig { + log.Println("Get config") + return golang.RegisterConfig{ + Name: "kaytu-io/plugin-gcp", + Version: version.VERSION, + Provider: "gcp", + Commands: []*golang.Command{ + { + Name: "compute-instance", + Description: "Get optimization suggestions for your Compute Engine Instances", + Flags: []*golang.Flag{ + { + Name: "profile", + Default: "", + Description: "GCP profile for authentication", + Required: false, + }, + }, + DefaultPreferences: preferences.DefaultComputeEnginePreferences, + LoginRequired: true, + }, + }, + } +} + +func (p *GCPPlugin) SetStream(stream golang.Plugin_RegisterClient) { + p.stream = stream +} + +// StartProcess implements sdk.Processor. +func (p *GCPPlugin) StartProcess(cmd string, flags map[string]string, kaytuAccessToken string, jobQueue *sdk.JobQueue) error { + + // scope used from https://developers.google.com/identity/protocols/oauth2/scopes#compute + gcpProvider := gcp.NewCompute( + []string{ + "https://www.googleapis.com/auth/compute.readonly", + }, + ) + + publishOptimizationItem := func(item *golang.OptimizationItem) { + p.stream.Send(&golang.PluginMessage{ + PluginMessage: &golang.PluginMessage_Oi{ + Oi: item, + }, + }) + } + + publishResultsReady := func(b bool) { + p.stream.Send(&golang.PluginMessage{ + PluginMessage: &golang.PluginMessage_Ready{ + Ready: &golang.ResultsReady{ + Ready: b, + }, + }, + }) + } + publishResultsReady(false) + + if cmd == "compute-instance" { + p.processor = compute_instance.NewComputeInstanceProcessor( + gcpProvider, + publishOptimizationItem, + kaytuAccessToken, + jobQueue, + ) + } else { + return fmt.Errorf("invalid command: %s", cmd) + } + jobQueue.SetOnFinish(func() { + publishResultsReady(true) + }) + + return nil +} + +func (p *GCPPlugin) ReEvaluate(evaluate *golang.ReEvaluate) { + p.processor.ReEvaluate(evaluate.Id, evaluate.Preferences) +} diff --git a/plugin/version/def.go b/plugin/version/def.go new file mode 100644 index 0000000..2232bbf --- /dev/null +++ b/plugin/version/def.go @@ -0,0 +1,3 @@ +package version + +var VERSION string diff --git a/utils/concurrent_map.go b/utils/concurrent_map.go new file mode 100644 index 0000000..5cc6e53 --- /dev/null +++ b/utils/concurrent_map.go @@ -0,0 +1,35 @@ +package util + +import "sync" + +type ConcurrentMap[K comparable, V any] struct { + data map[K]V + mutex sync.RWMutex +} + +func NewMap[K comparable, V any]() ConcurrentMap[K, V] { + return ConcurrentMap[K, V]{ + data: map[K]V{}, + mutex: sync.RWMutex{}, + } +} + +func (cm *ConcurrentMap[K, V]) Set(key K, value V) { + cm.mutex.Lock() + defer cm.mutex.Unlock() + cm.data[key] = value +} + +func (cm *ConcurrentMap[K, V]) Delete(key K) { + cm.mutex.Lock() + defer cm.mutex.Unlock() + delete(cm.data, key) +} + +func (cm *ConcurrentMap[K, V]) Get(key K) (V, bool) { + cm.mutex.RLock() + defer cm.mutex.RUnlock() + + v, ok := cm.data[key] + return v, ok +} From b63c6b828ba012b4643e162a0a674a654ca71f02 Mon Sep 17 00:00:00 2001 From: Adnan Gulegulzar Date: Fri, 7 Jun 2024 11:56:45 -0400 Subject: [PATCH 2/3] plugin now shows instance data on kaytu --- README.md | 2 +- .../processor/compute_instance/compute_instance.go | 4 ++-- .../compute_instance/job_compute_instance_list.go | 12 +++++++++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c30260a..7f426c9 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ curl -fsSL https://raw.githubusercontent.com/kaytu-io/kaytu/main/scripts/install Download Windows (Linux, and MacOS) binary from [releases](https://github.com/kaytu-io/kaytu/releases) -### 2. Login to AWS CLI +### 2. Login to Gcloud CLI Kaytu works with your existing Google Cloud CLI(gcloud) configuration (read-only access required) to gather metrics. diff --git a/plugin/processor/compute_instance/compute_instance.go b/plugin/processor/compute_instance/compute_instance.go index d694704..68253d6 100644 --- a/plugin/processor/compute_instance/compute_instance.go +++ b/plugin/processor/compute_instance/compute_instance.go @@ -12,7 +12,7 @@ import ( type ComputeInstanceProcessor struct { provider *gcp.Compute // metricProvider *gcp.CloudMonitoring - identification map[string]string + // identification map[string]string items util.ConcurrentMap[string, ComputeInstanceItem] publishOptimizationItem func(item *golang.OptimizationItem) kaytuAcccessToken string @@ -30,7 +30,7 @@ func NewComputeInstanceProcessor( provider: prv, // metricProvider: metric, // identification: identification, - // items: util.NewMap[string, EC2InstanceItem](), + items: util.NewMap[string, ComputeInstanceItem](), publishOptimizationItem: publishOptimizationItem, kaytuAcccessToken: kaytuAcccessToken, jobQueue: jobQueue, diff --git a/plugin/processor/compute_instance/job_compute_instance_list.go b/plugin/processor/compute_instance/job_compute_instance_list.go index 2331072..30d8f69 100644 --- a/plugin/processor/compute_instance/job_compute_instance_list.go +++ b/plugin/processor/compute_instance/job_compute_instance_list.go @@ -1,6 +1,7 @@ package compute_instance import ( + "context" "log" "strconv" @@ -30,11 +31,18 @@ func (job *ListComputeInstancesJob) Run() error { log.Println("Running list compute instance job") + err := job.processor.provider.InitializeClient(context.Background()) + if err != nil { + return err + } + instances, err := job.processor.provider.GetAllInstances() if err != nil { return err } + log.Printf("# of instances: %d", len(instances)) + for _, instance := range instances { oi := ComputeInstanceItem{ Name: *instance.Name, @@ -53,10 +61,12 @@ func (job *ListComputeInstancesJob) Run() error { // Preferences: preferences2.DefaultEC2Preferences, } + log.Printf("OI instance: %s", oi.Name) + job.processor.items.Set(oi.Id, oi) job.processor.publishOptimizationItem(oi.ToOptimizationItem()) } - return job.processor.provider.ListAllInstances() + return nil } From 64c715b57a89f52f3ff23e589aa3c2813d9c15f5 Mon Sep 17 00:00:00 2001 From: Adnan Gulegulzar Date: Fri, 7 Jun 2024 12:26:17 -0400 Subject: [PATCH 3/3] closing client --- .../processor/compute_instance/job_compute_instance_list.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugin/processor/compute_instance/job_compute_instance_list.go b/plugin/processor/compute_instance/job_compute_instance_list.go index 30d8f69..8daee62 100644 --- a/plugin/processor/compute_instance/job_compute_instance_list.go +++ b/plugin/processor/compute_instance/job_compute_instance_list.go @@ -67,6 +67,10 @@ func (job *ListComputeInstancesJob) Run() error { job.processor.publishOptimizationItem(oi.ToOptimizationItem()) } + if err = job.processor.provider.CloseClient(); err != nil { + return err + } + return nil }