Skip to content

Commit

Permalink
add wait group to speed up scrape
Browse files Browse the repository at this point in the history
  • Loading branch information
@LeoQuote
LeoQuote committed Sep 21, 2023
1 parent 772ebc2 commit e8205cf
Show file tree
Hide file tree
Showing 3 changed files with 144 additions and 523 deletions.
188 changes: 112 additions & 76 deletions exporter/cdn_exporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,39 +4,42 @@ import (
"aliyun-exporter/collector"
"fmt"
"github.com/aliyun/alibaba-cloud-sdk-go/services/cdn"
"strconv"

"github.com/aliyun/alibaba-cloud-sdk-go/services/cms"
"github.com/prometheus/client_golang/prometheus"
"strconv"
"sync"
)

const (
cdnnamespace = "aliyun"
)

type CdnExporter struct {
client *cms.Client
client *cms.Client
cdnClient *cdn.Client
rangeTime int64
delayTime int64

fluxHitRate *prometheus.Desc
hitRate *prometheus.Desc
backSourceBps *prometheus.Desc
BPS *prometheus.Desc
l1Acc *prometheus.Desc
backSourceAcc *prometheus.Desc
backSourceStatusRatio *prometheus.Desc
statusRatio *prometheus.Desc
rangeTime int64
delayTime int64
domains []string

fluxHitRate *prometheus.Desc
hitRate *prometheus.Desc
backSourceBps *prometheus.Desc
BPS *prometheus.Desc
l1Acc *prometheus.Desc
backSourceAcc *prometheus.Desc
backSourceStatusRatio *prometheus.Desc
statusRatio *prometheus.Desc
}

//实例化
// CdnCloudExporter 实例化
func CdnCloudExporter(cmsClient *cms.Client, cdnClient *cdn.Client, rangeTime int64, delayTime int64) *CdnExporter {
domains := collector.GetDomains(*cdnClient, "online")
return &CdnExporter{
client: cmsClient,
client: cmsClient,
cdnClient: cdnClient,
rangeTime: rangeTime,
delayTime: delayTime,
domains: domains,

fluxHitRate: prometheus.NewDesc(
prometheus.BuildFQName(cdnnamespace, "cdn", "flux_hit_rate"),
Expand Down Expand Up @@ -111,7 +114,7 @@ func CdnCloudExporter(cmsClient *cms.Client, cdnClient *cdn.Client, rangeTime in
}
}

//导出
// Describe 导出
func (e *CdnExporter) Describe(ch chan<- *prometheus.Desc) {
ch <- e.fluxHitRate
ch <- e.hitRate
Expand All @@ -123,13 +126,14 @@ func (e *CdnExporter) Describe(ch chan<- *prometheus.Desc) {
ch <- e.statusRatio
}

//收集
// Collect 收集
func (e *CdnExporter) Collect(ch chan<- prometheus.Metric) {
cdnDashboard := collector.NewCdnExporter(e.client)
var wg sync.WaitGroup

domains := collector.GetDomains(*e.cdnClient, "online")
//domains := []string{"vt3.doubanio.com"}
for _, domain := range domains {
for _, domain := range e.domains {
domain := domain
reqHitRate := collector.GetReqHitRate(*e.cdnClient, domain, e.rangeTime, e.delayTime)
// 去除掉数据量少的域名
if reqHitRate < 10 {
Expand All @@ -138,74 +142,106 @@ func (e *CdnExporter) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(
e.hitRate,
prometheus.GaugeValue,
float64(reqHitRate),
reqHitRate,
domain,
)
wg.Add(1)
go func() {
defer wg.Done()
statusProportion := collector.GetStatusCode(*e.cdnClient, domain, e.rangeTime, e.delayTime)
for status, proportion := range statusProportion {
proportion, _ = strconv.ParseFloat(fmt.Sprintf("%.3f", proportion), 64)
ch <- prometheus.MustNewConstMetric(
e.statusRatio,
prometheus.GaugeValue,
proportion,
domain,
status,
)
}
}()

wg.Add(1)
go func() {
defer wg.Done()
resourceStatus := collector.GetResourceStatusCode(*e.cdnClient, domain, e.rangeTime, e.delayTime)
for status, proportion := range resourceStatus {
proportion, _ = strconv.ParseFloat(fmt.Sprintf("%.3f", proportion), 64)
ch <- prometheus.MustNewConstMetric(
e.backSourceStatusRatio,
prometheus.GaugeValue,
proportion,
domain,
status,
)
}
}()

}

statusProportion := collector.GetStatusCode(*e.cdnClient, domain, e.rangeTime, e.delayTime)
for status, proportion := range statusProportion {
proportion, _ = strconv.ParseFloat(fmt.Sprintf("%.3f", proportion), 64)
wg.Add(1)
go func() {
defer wg.Done()
for _, point := range cdnDashboard.RetrieveHitRate(e.rangeTime, e.delayTime) {
ch <- prometheus.MustNewConstMetric(
e.statusRatio,
e.fluxHitRate,
prometheus.GaugeValue,
proportion,
domain,
status,
point.Average,
point.InstanceId,
)
}
}()

resourceStatus := collector.GetResourceStatusCode(*e.cdnClient, domain, e.rangeTime, e.delayTime)
for status, proportion := range resourceStatus {
proportion, _ = strconv.ParseFloat(fmt.Sprintf("%.3f", proportion), 64)
wg.Add(1)
go func() {
defer wg.Done()
for _, point := range cdnDashboard.RetrieveOriBps(e.rangeTime, e.delayTime) {
ch <- prometheus.MustNewConstMetric(
e.backSourceStatusRatio,
e.backSourceBps,
prometheus.GaugeValue,
proportion,
domain,
status,
point.Average/1000/1000,
point.InstanceId,
)
}
}
}()

for _, point := range cdnDashboard.RetrieveHitRate(e.rangeTime, e.delayTime) {
ch <- prometheus.MustNewConstMetric(
e.fluxHitRate,
prometheus.GaugeValue,
float64(point.Average),
point.InstanceId,
)
}
wg.Add(1)
go func() {
defer wg.Done()
for _, point := range cdnDashboard.RetrieveL1Acc(e.rangeTime, e.delayTime) {
ch <- prometheus.MustNewConstMetric(
e.l1Acc,
prometheus.GaugeValue,
point.Average,
point.InstanceId,
)
}
}()

for _, point := range cdnDashboard.RetrieveOriBps(e.rangeTime, e.delayTime) {
ch <- prometheus.MustNewConstMetric(
e.backSourceBps,
prometheus.GaugeValue,
float64(point.Average / 1000 / 1000),
point.InstanceId,
)
}
for _, point := range cdnDashboard.RetrieveL1Acc(e.rangeTime, e.delayTime) {
ch <- prometheus.MustNewConstMetric(
e.l1Acc,
prometheus.GaugeValue,
float64(point.Average),
point.InstanceId,
)
}
for _, point := range cdnDashboard.RetrieveOriAcc(e.rangeTime, e.delayTime) {
ch <- prometheus.MustNewConstMetric(
e.backSourceAcc,
prometheus.GaugeValue,
float64(point.Average),
point.InstanceId,
)
}
for _, point := range cdnDashboard.RetrieveBPS(e.rangeTime, e.delayTime) {
ch <- prometheus.MustNewConstMetric(
e.BPS,
prometheus.GaugeValue,
float64(point.Average / 1000 / 1000),
point.InstanceId,
)
}
wg.Add(1)
go func() {
defer wg.Done()
for _, point := range cdnDashboard.RetrieveOriAcc(e.rangeTime, e.delayTime) {
ch <- prometheus.MustNewConstMetric(
e.backSourceAcc,
prometheus.GaugeValue,
point.Average,
point.InstanceId,
)
}
}()

wg.Add(1)
go func() {
defer wg.Done()
for _, point := range cdnDashboard.RetrieveBPS(e.rangeTime, e.delayTime) {
ch <- prometheus.MustNewConstMetric(
e.BPS,
prometheus.GaugeValue,
point.Average/1000/1000,
point.InstanceId,
)
}
}()
wg.Wait()
}
18 changes: 9 additions & 9 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,22 @@ go 1.18

require (
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1695
github.com/prometheus/client_golang v1.12.2
github.com/prometheus/client_golang v1.16.0
)

require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
google.golang.org/protobuf v1.26.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
golang.org/x/sys v0.9.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/ini.v1 v1.66.2 // indirect
)
Loading

0 comments on commit e8205cf

Please sign in to comment.