From c04d593f582895460a04e67f5f7b0dfa99e7b168 Mon Sep 17 00:00:00 2001 From: hainenber Date: Wed, 13 Sep 2023 19:08:36 +0700 Subject: [PATCH 1/4] chore(build): switch to IBM/sarama --- component/loki/source/azure_event_hubs/azure_event_hubs.go | 2 +- .../loki/source/azure_event_hubs/internal/parser/parser.go | 2 +- .../source/azure_event_hubs/internal/parser/parser_test.go | 2 +- component/loki/source/internal/kafkatarget/authentication.go | 2 +- component/loki/source/internal/kafkatarget/config.go | 2 +- component/loki/source/internal/kafkatarget/consumer.go | 2 +- component/loki/source/internal/kafkatarget/consumer_test.go | 2 +- component/loki/source/internal/kafkatarget/kafkatarget.go | 2 +- component/loki/source/internal/kafkatarget/kafkatarget_test.go | 2 +- component/loki/source/internal/kafkatarget/oauth_provider.go | 2 +- component/loki/source/internal/kafkatarget/parser.go | 2 +- component/loki/source/internal/kafkatarget/target_syncer.go | 2 +- .../loki/source/internal/kafkatarget/target_syncer_test.go | 2 +- component/loki/source/kafka/kafka.go | 2 +- component/prometheus/exporter/kafka/kafka.go | 2 +- go.mod | 3 ++- go.sum | 2 ++ pkg/integrations/kafka_exporter/kafka_exporter.go | 2 +- 18 files changed, 20 insertions(+), 17 deletions(-) diff --git a/component/loki/source/azure_event_hubs/azure_event_hubs.go b/component/loki/source/azure_event_hubs/azure_event_hubs.go index a92d586f8f07..befe5cc0f4d0 100644 --- a/component/loki/source/azure_event_hubs/azure_event_hubs.go +++ b/component/loki/source/azure_event_hubs/azure_event_hubs.go @@ -6,7 +6,7 @@ import ( "net" "sync" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/go-kit/log/level" "github.com/grafana/agent/component" "github.com/grafana/agent/component/common/loki" diff --git a/component/loki/source/azure_event_hubs/internal/parser/parser.go b/component/loki/source/azure_event_hubs/internal/parser/parser.go index e82dbd37269d..7bda63b10fa5 100644 --- a/component/loki/source/azure_event_hubs/internal/parser/parser.go +++ b/component/loki/source/azure_event_hubs/internal/parser/parser.go @@ -11,7 +11,7 @@ import ( "strings" "time" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/grafana/agent/component/common/loki" "github.com/grafana/loki/pkg/logproto" "github.com/prometheus/common/model" diff --git a/component/loki/source/azure_event_hubs/internal/parser/parser_test.go b/component/loki/source/azure_event_hubs/internal/parser/parser_test.go index e3c2351955dc..3ade11700843 100644 --- a/component/loki/source/azure_event_hubs/internal/parser/parser_test.go +++ b/component/loki/source/azure_event_hubs/internal/parser/parser_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/relabel" "github.com/stretchr/testify/assert" diff --git a/component/loki/source/internal/kafkatarget/authentication.go b/component/loki/source/internal/kafkatarget/authentication.go index ffd7fc6bb512..99ec647bb692 100644 --- a/component/loki/source/internal/kafkatarget/authentication.go +++ b/component/loki/source/internal/kafkatarget/authentication.go @@ -11,7 +11,7 @@ import ( "github.com/xdg-go/scram" ) -// copied from https://github.com/Shopify/sarama/blob/44627b731c60bb90efe25573e7ef2b3f8df3fa23/examples/sasl_scram_client/scram_client.go +// copied from https://github.com/IBM/sarama/blob/44627b731c60bb90efe25573e7ef2b3f8df3fa23/examples/sasl_scram_client/scram_client.go var ( SHA256 scram.HashGeneratorFcn = sha256.New SHA512 scram.HashGeneratorFcn = sha512.New diff --git a/component/loki/source/internal/kafkatarget/config.go b/component/loki/source/internal/kafkatarget/config.go index a75ab30e6b5d..4fb81923cf61 100644 --- a/component/loki/source/internal/kafkatarget/config.go +++ b/component/loki/source/internal/kafkatarget/config.go @@ -1,7 +1,7 @@ package kafkatarget import ( - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/grafana/agent/component/common/loki" "github.com/grafana/dskit/flagext" promconfig "github.com/prometheus/common/config" diff --git a/component/loki/source/internal/kafkatarget/consumer.go b/component/loki/source/internal/kafkatarget/consumer.go index e945b83c5483..40eac7081ff7 100644 --- a/component/loki/source/internal/kafkatarget/consumer.go +++ b/component/loki/source/internal/kafkatarget/consumer.go @@ -10,7 +10,7 @@ import ( "sync" "time" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/grafana/dskit/backoff" diff --git a/component/loki/source/internal/kafkatarget/consumer_test.go b/component/loki/source/internal/kafkatarget/consumer_test.go index d5cf12362762..031850b04fc3 100644 --- a/component/loki/source/internal/kafkatarget/consumer_test.go +++ b/component/loki/source/internal/kafkatarget/consumer_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/go-kit/log" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" diff --git a/component/loki/source/internal/kafkatarget/kafkatarget.go b/component/loki/source/internal/kafkatarget/kafkatarget.go index da8b3b343f8f..b9c14f919742 100644 --- a/component/loki/source/internal/kafkatarget/kafkatarget.go +++ b/component/loki/source/internal/kafkatarget/kafkatarget.go @@ -8,7 +8,7 @@ import ( "fmt" "time" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/grafana/agent/component/common/loki" diff --git a/component/loki/source/internal/kafkatarget/kafkatarget_test.go b/component/loki/source/internal/kafkatarget/kafkatarget_test.go index 8a630384846e..777fd89480dc 100644 --- a/component/loki/source/internal/kafkatarget/kafkatarget_test.go +++ b/component/loki/source/internal/kafkatarget/kafkatarget_test.go @@ -13,7 +13,7 @@ import ( "github.com/grafana/agent/component/common/loki/client/fake" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/relabel" "github.com/stretchr/testify/require" diff --git a/component/loki/source/internal/kafkatarget/oauth_provider.go b/component/loki/source/internal/kafkatarget/oauth_provider.go index c2afeede3a45..43672a8f5791 100644 --- a/component/loki/source/internal/kafkatarget/oauth_provider.go +++ b/component/loki/source/internal/kafkatarget/oauth_provider.go @@ -8,7 +8,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" ) func NewOAuthProvider(opts OAuthConfig) (sarama.AccessTokenProvider, error) { diff --git a/component/loki/source/internal/kafkatarget/parser.go b/component/loki/source/internal/kafkatarget/parser.go index c6234a93a804..325d9582cd1d 100644 --- a/component/loki/source/internal/kafkatarget/parser.go +++ b/component/loki/source/internal/kafkatarget/parser.go @@ -1,7 +1,7 @@ package kafkatarget import ( - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/grafana/agent/component/common/loki" "github.com/grafana/loki/pkg/logproto" "github.com/prometheus/common/model" diff --git a/component/loki/source/internal/kafkatarget/target_syncer.go b/component/loki/source/internal/kafkatarget/target_syncer.go index cbe36a334e05..4179d993852b 100644 --- a/component/loki/source/internal/kafkatarget/target_syncer.go +++ b/component/loki/source/internal/kafkatarget/target_syncer.go @@ -11,7 +11,7 @@ import ( "sync" "time" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/go-kit/log" "github.com/go-kit/log/level" promconfig "github.com/prometheus/common/config" diff --git a/component/loki/source/internal/kafkatarget/target_syncer_test.go b/component/loki/source/internal/kafkatarget/target_syncer_test.go index 005475d96b52..8c1b755b4cde 100644 --- a/component/loki/source/internal/kafkatarget/target_syncer_test.go +++ b/component/loki/source/internal/kafkatarget/target_syncer_test.go @@ -12,7 +12,7 @@ import ( "github.com/grafana/dskit/flagext" "github.com/prometheus/common/config" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/go-kit/log" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/relabel" diff --git a/component/loki/source/kafka/kafka.go b/component/loki/source/kafka/kafka.go index 0bc09f44f51d..4f2e2516c9f9 100644 --- a/component/loki/source/kafka/kafka.go +++ b/component/loki/source/kafka/kafka.go @@ -4,7 +4,7 @@ import ( "context" "sync" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/go-kit/log/level" "github.com/grafana/agent/component" "github.com/grafana/agent/component/common/config" diff --git a/component/prometheus/exporter/kafka/kafka.go b/component/prometheus/exporter/kafka/kafka.go index 74edcd782b0e..00f2ce589a2f 100644 --- a/component/prometheus/exporter/kafka/kafka.go +++ b/component/prometheus/exporter/kafka/kafka.go @@ -3,7 +3,7 @@ package kafka import ( "fmt" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/grafana/agent/component" "github.com/grafana/agent/component/discovery" "github.com/grafana/agent/component/prometheus/exporter" diff --git a/go.mod b/go.mod index 4f087e394d44..6bb5967f0aea 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,10 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/Azure/go-autorest/autorest v0.11.29 + github.com/IBM/sarama v1.40.1 github.com/Lusitaniae/apache_exporter v0.11.1-0.20220518131644-f9522724dab4 github.com/Masterminds/sprig/v3 v3.2.3 github.com/PuerkitoBio/rehttp v1.1.0 - github.com/Shopify/sarama v1.38.1 github.com/alecthomas/kingpin/v2 v2.3.2 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 github.com/aws/aws-sdk-go v1.44.327 @@ -624,6 +624,7 @@ require ( ) require ( + github.com/Shopify/sarama v1.38.1 // indirect github.com/drone/envsubst v1.0.3 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/leoluk/perflib_exporter v0.2.0 // indirect diff --git a/go.sum b/go.sum index a0ab05b4f239..07251df97a21 100644 --- a/go.sum +++ b/go.sum @@ -524,6 +524,8 @@ github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 h1:KeNholpO2xKjgaa github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962/go.mod h1:kC29dT1vFpj7py2OvG1khBdQpo3kInWP+6QipLbdngo= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= +github.com/IBM/sarama v1.40.1 h1:lL01NNg/iBeigUbT+wpPysuTYW6roHo6kc1QrffRf0k= +github.com/IBM/sarama v1.40.1/go.mod h1:+5OFwA5Du9I6QrznhaMHsuwWdWZNMjaBSIxEWEgKOYE= github.com/Jeffail/gabs v1.1.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= diff --git a/pkg/integrations/kafka_exporter/kafka_exporter.go b/pkg/integrations/kafka_exporter/kafka_exporter.go index e9d959508e7b..b8cc491b120a 100644 --- a/pkg/integrations/kafka_exporter/kafka_exporter.go +++ b/pkg/integrations/kafka_exporter/kafka_exporter.go @@ -5,7 +5,7 @@ import ( config_util "github.com/prometheus/common/config" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" kafka_exporter "github.com/davidmparrott/kafka_exporter/v2/exporter" "github.com/go-kit/log" "github.com/grafana/agent/pkg/integrations" From 0fd690221e54abeb9ca02bae4e02b2ff7d7a5e3b Mon Sep 17 00:00:00 2001 From: hainenber Date: Wed, 13 Sep 2023 19:09:59 +0700 Subject: [PATCH 2/4] doc(CHANGELOG): add new entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e89ac0f11174..515f586a4042 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,7 @@ Main (unreleased) have been standardized. The first fields will always be `ts`, `level`, and `msg`, followed by non-common fields. Previously, the position of `msg` was not consistent. (@rfratto) +- Switch to `IBM/sarama` module. (@hainenber) ### Bugfixes From fba88be1dee437c92e04e51d8eba81d2fd1cb01d Mon Sep 17 00:00:00 2001 From: hainenber Date: Wed, 13 Sep 2023 20:10:40 +0700 Subject: [PATCH 3/4] fix(loki/kafkatarget): use new sarama method instead of deprecated ones --- component/loki/source/internal/kafkatarget/target_syncer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/component/loki/source/internal/kafkatarget/target_syncer.go b/component/loki/source/internal/kafkatarget/target_syncer.go index 4179d993852b..027ac5fd7073 100644 --- a/component/loki/source/internal/kafkatarget/target_syncer.go +++ b/component/loki/source/internal/kafkatarget/target_syncer.go @@ -65,11 +65,11 @@ func NewSyncer( switch cfg.KafkaConfig.Assignor { case sarama.StickyBalanceStrategyName: - config.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategySticky + config.Consumer.Group.Rebalance.Strategy = sarama.NewBalanceStrategyRange() case sarama.RoundRobinBalanceStrategyName: - config.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRoundRobin + config.Consumer.Group.Rebalance.Strategy = sarama.NewBalanceStrategyRoundRobin() case sarama.RangeBalanceStrategyName, "": - config.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRange + config.Consumer.Group.Rebalance.Strategy = sarama.NewBalanceStrategyRange() default: return nil, fmt.Errorf("unrecognized consumer group partition assignor: %s", cfg.KafkaConfig.Assignor) } From 488dec25d011edb858657467892be8338d90f8bc Mon Sep 17 00:00:00 2001 From: hainenber Date: Wed, 13 Sep 2023 20:19:24 +0700 Subject: [PATCH 4/4] fix(loki/kafkatarget): correct sticky balance strategy --- component/loki/source/internal/kafkatarget/target_syncer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/component/loki/source/internal/kafkatarget/target_syncer.go b/component/loki/source/internal/kafkatarget/target_syncer.go index 027ac5fd7073..0355c6134329 100644 --- a/component/loki/source/internal/kafkatarget/target_syncer.go +++ b/component/loki/source/internal/kafkatarget/target_syncer.go @@ -65,7 +65,7 @@ func NewSyncer( switch cfg.KafkaConfig.Assignor { case sarama.StickyBalanceStrategyName: - config.Consumer.Group.Rebalance.Strategy = sarama.NewBalanceStrategyRange() + config.Consumer.Group.Rebalance.Strategy = sarama.NewBalanceStrategySticky() case sarama.RoundRobinBalanceStrategyName: config.Consumer.Group.Rebalance.Strategy = sarama.NewBalanceStrategyRoundRobin() case sarama.RangeBalanceStrategyName, "":