Skip to content

Commit

Permalink
Merge pull request #282 from intelops/pvc
Browse files Browse the repository at this point in the history 
Pvc
  • Loading branch information
@vijeyash1
vijeyash1 committed Dec 13, 2023
2 parents 220a3d2 + 318eadb commit f8691fa
Show file tree
Hide file tree
Showing 6 changed files with 67 additions and 79 deletions.
11 changes: 6 additions & 5 deletions agent/kubviz/kubePreUpgrade.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,19 +79,20 @@ func publishK8sDepricated_Deleted_Api(result *model.Result, js nats.JetStreamCon
}

func KubePreUpgradeDetector(config *rest.Config, js nats.JetStreamContext) error {
swaggerdir, err := os.MkdirTemp("", "kubepug")
pvcMountPath := "/mnt/agent/kbz"
uniqueDir := fmt.Sprintf("%s/kubepug", pvcMountPath)
err := os.MkdirAll(uniqueDir, 0755)
if err != nil {
return err
}
filename := fmt.Sprintf("%s/swagger-%s.json", swaggerdir, k8sVersion)
filename := fmt.Sprintf("%s/swagger-%s.json", uniqueDir, k8sVersion)
url := fmt.Sprintf("%s/%s/%s", baseURL, k8sVersion, fileURL)
err = downloadFile(filename, url)
if err != nil {
return err
}
defer os.RemoveAll(swaggerdir)
swaggerfile := filename
kubernetesAPIs, err := PopulateKubeAPIMap(swaggerfile)
defer os.RemoveAll(filename)
kubernetesAPIs, err := PopulateKubeAPIMap(filename)
if err != nil {
return err
}
Expand Down
11 changes: 10 additions & 1 deletion agent/kubviz/trivy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ package main
import (
"bytes"
"encoding/json"
"fmt"
"log"
"os"
exec "os/exec"
"strings"

Expand All @@ -29,8 +31,15 @@ func executeCommandTrivy(command string) ([]byte, error) {
return outc.Bytes(), err
}
func RunTrivyK8sClusterScan(js nats.JetStreamContext) error {
pvcMountPath := "/mnt/agent/kbz"
trivyCacheDir := fmt.Sprintf("%s/trivy-cache", pvcMountPath)
err := os.MkdirAll(trivyCacheDir, 0755)
if err != nil {
log.Printf("Error creating Trivy cache directory: %v\n", err)
return err
}
var report report.ConsolidatedReport
cmdString := "trivy k8s --report summary cluster --exclude-nodes kubernetes.io/arch:amd64 --timeout 60m -f json --cache-dir /tmp/.cache --debug"
cmdString := fmt.Sprintf("trivy k8s --report summary cluster --exclude-nodes kubernetes.io/arch:amd64 --timeout 60m -f json --cache-dir %s --debug", trivyCacheDir)
clearCacheCmd := "trivy k8s --clear-cache"
out, err := executeCommandTrivy(cmdString)
if err != nil {
Expand Down
20 changes: 10 additions & 10 deletions agent/kubviz/trivy_image.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package main

import (
"encoding/json"
"fmt"
"log"
"os"
"strings"
Expand All @@ -15,6 +16,13 @@ import (
)

func RunTrivyImageScans(config *rest.Config, js nats.JetStreamContext) error {
pvcMountPath := "/mnt/agent/kbz"
trivyImageCacheDir := fmt.Sprintf("%s/trivy-imagecache", pvcMountPath)
err := os.MkdirAll(trivyImageCacheDir, 0755)
if err != nil {
log.Printf("Error creating Trivy Image cache directory: %v\n", err)
return err
}
clearCacheCmd := "trivy image --clear-cache"

images, err := ListImages(config)
Expand All @@ -25,7 +33,8 @@ func RunTrivyImageScans(config *rest.Config, js nats.JetStreamContext) error {

for _, image := range images {
var report types.Report
out, err := executeCommand("trivy image " + image.PullableImage + " --timeout 60m -f json -q --cache-dir /tmp/.cache")
scanCmd := fmt.Sprintf("trivy image %s --timeout 60m -f json -q --cache-dir %s", image.PullableImage, trivyImageCacheDir)
out, err := executeCommand(scanCmd)
if err != nil {
log.Printf("Error scanning image %s: %v", image.PullableImage, err)
continue // Move on to the next image in case of an error
Expand Down Expand Up @@ -74,12 +83,3 @@ func publishImageScanReports(report types.Report, js nats.JetStreamContext) erro
log.Printf("Trivy image report with ID:%s has been published\n", metrics.ID)
return nil
}

func cleanupCache(cacheDir string) {
err := os.RemoveAll(cacheDir)
if err != nil {
log.Printf("Failed to clean up cache directory %s: %v", cacheDir, err)
} else {
log.Printf("Cache directory %s cleaned up successfully", cacheDir)
}
}
12 changes: 10 additions & 2 deletions agent/kubviz/trivy_sbom.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"log"
"os"
"os/exec"

"github.com/aquasecurity/trivy/pkg/sbom/cyclonedx"
Expand Down Expand Up @@ -46,6 +47,13 @@ func executeCommandSbom(command string) ([]byte, error) {
}

func RunTrivySbomScan(config *rest.Config, js nats.JetStreamContext) error {
pvcMountPath := "/mnt/agent/kbz"
trivySbomCacheDir := fmt.Sprintf("%s/trivy-sbomcache", pvcMountPath)
err := os.MkdirAll(trivySbomCacheDir, 0755)
if err != nil {
log.Printf("Error creating Trivy cache directory: %v\n", err)
return err
}
clearCacheCmd := "trivy image --clear-cache"

log.Println("trivy sbom run started")
Expand All @@ -56,8 +64,8 @@ func RunTrivySbomScan(config *rest.Config, js nats.JetStreamContext) error {
}
for _, image := range images {

command := fmt.Sprintf("trivy image --format cyclonedx %s %s", image.PullableImage, "--cache-dir /tmp/.cache")
out, err := executeCommandSbom(command)
sbomcmd := fmt.Sprintf("trivy image --format cyclonedx %s --cache-dir %s", image.PullableImage, trivySbomCacheDir)
out, err := executeCommandSbom(sbomcmd)

if err != nil {
log.Printf("Error executing Trivy for image sbom %s: %v", image.PullableImage, err)
Expand Down
90 changes: 30 additions & 60 deletions client/pkg/clickhouse/db_client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,14 +136,12 @@ func (c *DBClient) InsertContainerEventAzure(pushEvent model.AzureContainerPushE

tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}

stmt, err := tx.Prepare(string(InsertAzureContainerPushEvent))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand Down Expand Up @@ -187,14 +185,12 @@ func (c *DBClient) InsertContainerEventAzure(pushEvent model.AzureContainerPushE
func (c *DBClient) InsertContainerEventQuay(pushEvent model.QuayImagePushPayload) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}

stmt, err := tx.Prepare(string(InsertQuayContainerPushEvent))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand Down Expand Up @@ -240,14 +236,12 @@ func (c *DBClient) InsertContainerEventQuay(pushEvent model.QuayImagePushPayload
func (c *DBClient) InsertContainerEventJfrog(pushEvent model.JfrogContainerPushEventPayload) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}

stmt, err := tx.Prepare(string(InsertJfrogContainerPushEvent))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand Down Expand Up @@ -293,13 +287,11 @@ func (c *DBClient) InsertContainerEventJfrog(pushEvent model.JfrogContainerPushE
func (c *DBClient) InsertRakeesMetrics(metrics model.RakeesMetrics) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(string(InsertRakees))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand All @@ -325,13 +317,11 @@ func (c *DBClient) InsertRakeesMetrics(metrics model.RakeesMetrics) {
func (c *DBClient) InsertKetallEvent(metrics model.Resource) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(string(InsertKetall))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand All @@ -356,13 +346,11 @@ func (c *DBClient) InsertKetallEvent(metrics model.Resource) {
func (c *DBClient) InsertOutdatedEvent(metrics model.CheckResultfinal) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(string(InsertOutdated))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand All @@ -389,13 +377,11 @@ func (c *DBClient) InsertOutdatedEvent(metrics model.CheckResultfinal) {
func (c *DBClient) InsertDeprecatedAPI(deprecatedAPI model.DeprecatedAPI) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(string(InsertDepricatedApi))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand Down Expand Up @@ -429,13 +415,11 @@ func (c *DBClient) InsertDeprecatedAPI(deprecatedAPI model.DeprecatedAPI) {
func (c *DBClient) InsertDeletedAPI(deletedAPI model.DeletedAPI) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(string(InsertDeletedApi))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand Down Expand Up @@ -470,13 +454,11 @@ func (c *DBClient) InsertDeletedAPI(deletedAPI model.DeletedAPI) {
func (c *DBClient) InsertKubvizEvent(metrics model.Metrics) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(string(InsertKubvizEvent))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand Down Expand Up @@ -539,13 +521,11 @@ func (c *DBClient) InsertContainerEvent(event string) {
func (c *DBClient) InsertKubeScoreMetrics(metrics model.KubeScoreRecommendations) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(InsertKubeScore)
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}
defer stmt.Close()

Expand All @@ -571,13 +551,11 @@ func (c *DBClient) InsertTrivyMetrics(metrics model.Trivy) {
for _, vulnerability := range result.Vulnerabilities {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(InsertTrivyVul)
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}
if _, err := stmt.Exec(
metrics.ID,
Expand Down Expand Up @@ -608,13 +586,11 @@ func (c *DBClient) InsertTrivyMetrics(metrics model.Trivy) {
for _, misconfiguration := range result.Misconfigurations {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(InsertTrivyMisconfig)
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand Down Expand Up @@ -654,13 +630,11 @@ func (c *DBClient) InsertTrivyImageMetrics(metrics model.TrivyImage) {
for _, vulnerability := range result.Vulnerabilities {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(InsertTrivyImage)
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

if _, err := stmt.Exec(
Expand Down Expand Up @@ -700,13 +674,11 @@ func (c *DBClient) InsertTrivySbomMetrics(metrics model.Sbom) {
if result.CycloneDX != nil {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(InsertTrivySbom)
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

if _, err := stmt.Exec(
Expand Down Expand Up @@ -834,14 +806,12 @@ func (c *DBClient) RetrieveKubvizEvent() ([]model.DbEvent, error) {
func (c *DBClient) InsertContainerEventDockerHub(build model.DockerHubBuild) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(string(InsertDockerHubBuild))

if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}
defer stmt.Close()

Expand Down
2 changes: 1 addition & 1 deletion dockerfiles/client/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o k8smetri

# Use distroless as minimal base image to package the manager binary
# Refer to https://github.com/GoogleContainerTools/distroless for more details
FROM gcr.io/distroless/static:nonroot
FROM golang:alpine
WORKDIR /
COPY --from=builder /workspace/k8smetrics_client .
USER 65532:65532
Expand Down

0 comments on commit f8691fa

Please sign in to comment.