Skip to content

Commit 5e033c0

Browse files
cbullingercbullinger
authored
Merge pull request #53 from cbullinger/docsp-46224-46228
DOCSP-46224: Add archive examples for Scalability page (Arch Center)
2 parents 3777430 + 28416f9 commit 5e033c0

68 files changed

Lines changed: 2298 additions & 485 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎generated-usage-examples/go/atlas-sdk-go/main.snippet.archive-collections.go‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
// See entire project at https://github.com/mongodb/atlas-architecture-go-sdk
2+
package main
3+
4+
import (
5+
"context"
6+
"fmt"
7+
"log"
8+
"time"
9+
10+
"atlas-sdk-go/internal/archive"
11+
"atlas-sdk-go/internal/auth"
12+
"atlas-sdk-go/internal/config"
13+
14+
"github.com/joho/godotenv"
15+
)
16+
17+
func main() {
18+
envFile := ".env.production"
19+
if err := godotenv.Load(envFile); err != nil {
20+
log.Printf("Warning: could not load %s file: %v", envFile, err)
21+
}
22+
23+
secrets, cfg, err := config.LoadAllFromEnv()
24+
if err != nil {
25+
log.Fatalf("Failed to load configuration %v", err)
26+
}
27+
28+
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
29+
defer cancel()
30+
client, err := auth.NewClient(ctx, cfg, secrets)
31+
if err != nil {
32+
log.Fatalf("Failed to initialize authentication client: %v", err)
33+
}
34+
35+
projectID := cfg.ProjectID
36+
if projectID == "" {
37+
log.Fatal("Failed to find Project ID in configuration")
38+
}
39+
40+
fmt.Printf("Starting archive analysis for project: %s\n", projectID)
41+
42+
// Get all clusters in the project
43+
clusters, _, err := client.ClustersApi.ListClusters(ctx, projectID).Execute()
44+
if err != nil {
45+
log.Fatalf("Failed to list clusters: %v", err)
46+
}
47+
48+
fmt.Printf("\nFound %d clusters to analyze\n", len(clusters.GetResults()))
49+
50+
// Connect to each cluster and analyze collections for archiving
51+
failedArchives := 0
52+
skippedCandidates := 0
53+
totalCandidates := 0
54+
55+
// Create archive options with custom settings
56+
opts := archive.DefaultOptions()
57+
opts.DefaultRetentionMultiplier = 2
58+
opts.MinimumRetentionDays = 30
59+
opts.EnableDataExpiration = true
60+
opts.ArchiveSchedule = "DAILY"
61+
62+
for _, cluster := range clusters.GetResults() {
63+
clusterName := cluster.GetName()
64+
fmt.Printf("\n=== Analyzing cluster: %s ===", clusterName)
65+
66+
// Find collections suitable for archiving based on demo criteria.
67+
// This simplified example first selects all collections with counts, and then filters them.
68+
// NOTE: In a real implementation, you would analyze collections based on size, age,
69+
// access patterns, and other factors to determine candidates for archiving.
70+
stats := archive.ListCollectionsWithCounts(ctx, client, projectID, clusterName)
71+
candidates := make([]archive.Candidate, 0)
72+
const docThreshold = 100000
73+
for _, s := range stats {
74+
// Skip internal databases
75+
if s.DatabaseName == "admin" || s.DatabaseName == "local" || s.DatabaseName == "config" {
76+
continue
77+
}
78+
// Demo criterion: collections with >= 100k documents
79+
if s.EstimatedCount >= docThreshold {
80+
candidates = append(candidates, archive.Candidate{
81+
DatabaseName: s.DatabaseName,
82+
CollectionName: s.CollectionName,
83+
DateField: "createdAt",
84+
DateFormat: "DATE",
85+
RetentionDays: 90,
86+
PartitionFields: []string{"createdAt"},
87+
})
88+
}
89+
}
90+
totalCandidates += len(candidates)
91+
fmt.Printf("\nFound %d collections eligible for archiving in cluster %s\n",
92+
len(candidates), clusterName)
93+
94+
// Configure online archive for each candidate collection
95+
for _, candidate := range candidates {
96+
// Pre-validate candidate before attempting configuration
97+
if err := archive.ValidateCandidate(candidate, opts); err != nil {
98+
fmt.Printf("- Skipping %s.%s: invalid candidate: %v\n",
99+
candidate.DatabaseName, candidate.CollectionName, err)
100+
skippedCandidates++
101+
continue
102+
}
103+
104+
fmt.Printf("- Configuring archive for %s.%s\n",
105+
candidate.DatabaseName, candidate.CollectionName)
106+
107+
configureErr := archive.ConfigureOnlineArchive(ctx, client, projectID, clusterName, candidate, opts)
108+
if configureErr != nil {
109+
fmt.Printf(" Failed to configure archive: %v\n", configureErr)
110+
failedArchives++
111+
continue
112+
}
113+
114+
fmt.Printf(" Successfully configured online archive for %s.%s\n",
115+
candidate.DatabaseName, candidate.CollectionName)
116+
}
117+
}
118+
119+
if skippedCandidates > 0 {
120+
fmt.Printf("\nINFO: Skipped %d of %d candidates due to validation errors\n", skippedCandidates, totalCandidates)
121+
}
122+
if failedArchives > 0 {
123+
fmt.Printf("WARNING: %d of %d archive configurations failed (excluding skipped)\n", failedArchives, totalCandidates-skippedCandidates)
124+
}
125+
126+
fmt.Println("Archive analysis and configuration completed.")
127+
}
128+

‎generated-usage-examples/go/atlas-sdk-go/main.snippet.get-logs.go‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import (
88

99
"atlas-sdk-go/internal/auth"
1010
"atlas-sdk-go/internal/config"
11-
"atlas-sdk-go/internal/errors"
1211
"atlas-sdk-go/internal/fileutils"
1312
"atlas-sdk-go/internal/logs"
1413

@@ -17,55 +16,58 @@ import (
1716
)
1817

1918
func main() {
20-
if err := godotenv.Load(); err != nil {
21-
log.Printf("Warning: .env file not loaded: %v", err)
19+
envFile := ".env.production"
20+
if err := godotenv.Load(envFile); err != nil {
21+
log.Printf("Warning: could not load %s file: %v", envFile, err)
2222
}
2323

24-
secrets, cfg, err := config.LoadAll("configs/config.json")
24+
secrets, cfg, err := config.LoadAllFromEnv()
2525
if err != nil {
26-
errors.ExitWithError("Failed to load configuration", err)
26+
log.Fatalf("Failed to load configuration %v", err)
2727
}
2828

29-
client, err := auth.NewClient(cfg, secrets)
29+
ctx := context.Background()
30+
client, err := auth.NewClient(ctx, cfg, secrets)
3031
if err != nil {
31-
errors.ExitWithError("Failed to initialize authentication client", err)
32+
log.Fatalf("Failed to initialize authentication client: %v", err)
3233
}
3334

34-
ctx := context.Background()
35-
3635
// Fetch logs with the provided parameters
3736
p := &admin.GetHostLogsApiParams{
3837
GroupId: cfg.ProjectID,
3938
HostName: cfg.HostName,
4039
LogName: "mongodb",
4140
}
41+
fmt.Printf("Request parameters: GroupID=%s, HostName=%s, LogName=%s\n",
42+
cfg.ProjectID, cfg.HostName, p.LogName)
4243
rc, err := logs.FetchHostLogs(ctx, client.MonitoringAndLogsApi, p)
4344
if err != nil {
44-
errors.ExitWithError("Failed to fetch logs", err)
45+
log.Fatalf("Failed to fetch logs: %v", err)
4546
}
4647
defer fileutils.SafeClose(rc)
4748

4849
// Prepare output paths
50+
// If the ATLAS_DOWNLOADS_DIR env variable is set, it will be used as the base directory for output files
4951
outDir := "logs"
5052
prefix := fmt.Sprintf("%s_%s", p.HostName, p.LogName)
5153
gzPath, err := fileutils.GenerateOutputPath(outDir, prefix, "gz")
5254
if err != nil {
53-
errors.ExitWithError("Failed to generate GZ output path", err)
55+
log.Fatalf("Failed to generate GZ output path: %v", err)
5456
}
5557
txtPath, err := fileutils.GenerateOutputPath(outDir, prefix, "txt")
5658
if err != nil {
57-
errors.ExitWithError("Failed to generate TXT output path", err)
59+
log.Fatalf("Failed to generate TXT output path: %v", err)
5860
}
5961

6062
// Save compressed logs
6163
if err := fileutils.WriteToFile(rc, gzPath); err != nil {
62-
errors.ExitWithError("Failed to save compressed logs", err)
64+
log.Fatalf("Failed to save compressed logs: %v", err)
6365
}
6466
fmt.Println("Saved compressed log to", gzPath)
6567

6668
// Decompress logs
6769
if err := fileutils.DecompressGzip(gzPath, txtPath); err != nil {
68-
errors.ExitWithError("Failed to decompress logs", err)
70+
log.Fatalf("Failed to decompress logs: %v", err)
6971
}
7072
fmt.Println("Uncompressed log to", txtPath)
7173
}

‎generated-usage-examples/go/atlas-sdk-go/main.snippet.get-metrics-dev.go‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,30 +9,29 @@ import (
99

1010
"atlas-sdk-go/internal/auth"
1111
"atlas-sdk-go/internal/config"
12-
"atlas-sdk-go/internal/errors"
1312
"atlas-sdk-go/internal/metrics"
1413

1514
"github.com/joho/godotenv"
1615
"go.mongodb.org/atlas-sdk/v20250219001/admin"
1716
)
1817

1918
func main() {
20-
if err := godotenv.Load(); err != nil {
21-
log.Printf("Warning: .env file not loaded: %v", err)
19+
envFile := ".env.development"
20+
if err := godotenv.Load(envFile); err != nil {
21+
log.Printf("Warning: could not load %s file: %v", envFile, err)
2222
}
2323

24-
secrets, cfg, err := config.LoadAll("configs/config.json")
24+
secrets, cfg, err := config.LoadAllFromEnv()
2525
if err != nil {
26-
errors.ExitWithError("Failed to load configuration", err)
26+
log.Fatalf("Failed to load configuration %v", err)
2727
}
2828

29-
client, err := auth.NewClient(cfg, secrets)
29+
ctx := context.Background()
30+
client, err := auth.NewClient(ctx, cfg, secrets)
3031
if err != nil {
31-
errors.ExitWithError("Failed to initialize authentication client", err)
32+
log.Fatalf("Failed to initialize authentication client: %v", err)
3233
}
3334

34-
ctx := context.Background()
35-
3635
// Fetch disk metrics with the provided parameters
3736
p := &admin.GetDiskMeasurementsApiParams{
3837
GroupId: cfg.ProjectID,
@@ -44,13 +43,13 @@ func main() {
4443
}
4544
view, err := metrics.FetchDiskMetrics(ctx, client.MonitoringAndLogsApi, p)
4645
if err != nil {
47-
errors.ExitWithError("Failed to fetch disk metrics", err)
46+
log.Fatalf("Failed to fetch disk metrics: %v", err)
4847
}
4948

5049
// Output metrics
5150
out, err := json.MarshalIndent(view, "", " ")
5251
if err != nil {
53-
errors.ExitWithError("Failed to format metrics data", err)
52+
log.Fatalf("Failed to format metrics data: %v", err)
5453
}
5554
fmt.Println(string(out))
5655
}

‎generated-usage-examples/go/atlas-sdk-go/main.snippet.get-metrics-prod.go‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@ import (
77
"fmt"
88
"log"
99

10-
"atlas-sdk-go/internal/errors"
11-
1210
"atlas-sdk-go/internal/auth"
1311
"atlas-sdk-go/internal/config"
1412

@@ -19,22 +17,22 @@ import (
1917
)
2018

2119
func main() {
22-
if err := godotenv.Load(); err != nil {
23-
log.Printf("Warning: .env file not loaded: %v", err)
20+
envFile := ".env.production"
21+
if err := godotenv.Load(envFile); err != nil {
22+
log.Printf("Warning: could not load %s file: %v", envFile, err)
2423
}
2524

26-
secrets, cfg, err := config.LoadAll("configs/config.json")
25+
secrets, cfg, err := config.LoadAllFromEnv()
2726
if err != nil {
28-
errors.ExitWithError("Failed to load configuration", err)
27+
log.Fatalf("Failed to load configuration %v", err)
2928
}
3029

31-
client, err := auth.NewClient(cfg, secrets)
30+
ctx := context.Background()
31+
client, err := auth.NewClient(ctx, cfg, secrets)
3232
if err != nil {
33-
errors.ExitWithError("Failed to initialize authentication client", err)
33+
log.Fatalf("Failed to initialize authentication client: %v", err)
3434
}
3535

36-
ctx := context.Background()
37-
3836
// Fetch process metrics with the provided parameters
3937
p := &admin.GetHostMeasurementsApiParams{
4038
GroupId: cfg.ProjectID,
@@ -52,13 +50,13 @@ func main() {
5250

5351
view, err := metrics.FetchProcessMetrics(ctx, client.MonitoringAndLogsApi, p)
5452
if err != nil {
55-
errors.ExitWithError("Failed to fetch process metrics", err)
53+
log.Fatalf("Failed to fetch process metrics: %v", err)
5654
}
5755

5856
// Output metrics
5957
out, err := json.MarshalIndent(view, "", " ")
6058
if err != nil {
61-
errors.ExitWithError("Failed to format metrics data", err)
59+
log.Fatalf("Failed to format metrics data: %v", err)
6260
}
6361
fmt.Println(string(out))
6462
}

0 commit comments

Comments
 (0)