From 065fedbef4d2b83196143a17526c83de1a2a7901 Mon Sep 17 00:00:00 2001 From: Vyacheslav Starostin <32613074+vstarostin@users.noreply.github.com> Date: Thu, 16 May 2024 16:03:14 +0500 Subject: [PATCH 01/16] sonarExecuteScan: update documentation (#4930) * sonarExecuteScan: update documentation * Jenkins only * Jenkins only --- cmd/sonarExecuteScan_generated.go | 2 +- resources/metadata/sonarExecuteScan.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/sonarExecuteScan_generated.go b/cmd/sonarExecuteScan_generated.go index 9f2350c4f4..c818d61499 100644 --- a/cmd/sonarExecuteScan_generated.go +++ b/cmd/sonarExecuteScan_generated.go @@ -246,7 +246,7 @@ func SonarExecuteScanCommand() *cobra.Command { func addSonarExecuteScanFlags(cmd *cobra.Command, stepConfig *sonarExecuteScanOptions) { cmd.Flags().StringVar(&stepConfig.Instance, "instance", os.Getenv("PIPER_instance"), "Jenkins only: The name of the SonarQube instance defined in the Jenkins settings. DEPRECATED: use serverUrl parameter instead") cmd.Flags().StringVar(&stepConfig.Proxy, "proxy", os.Getenv("PIPER_proxy"), "Proxy URL to be used for communication with the SonarQube instance.") - cmd.Flags().StringVar(&stepConfig.ServerURL, "serverUrl", os.Getenv("PIPER_serverUrl"), "The URL to the Sonar backend.") + cmd.Flags().StringVar(&stepConfig.ServerURL, "serverUrl", os.Getenv("PIPER_serverUrl"), "The URL to the Sonar backend. Jenkins only: The serverUrl parameter requires the [`instance`](#instance) parameter to be explicitly set to an empty string, as it will have no effect otherwise.") cmd.Flags().StringVar(&stepConfig.Token, "token", os.Getenv("PIPER_token"), "Token used to authenticate with the Sonar Server.") cmd.Flags().StringVar(&stepConfig.Organization, "organization", os.Getenv("PIPER_organization"), "SonarCloud.io only: Organization that the project will be assigned to in SonarCloud.io.") cmd.Flags().StringSliceVar(&stepConfig.CustomTLSCertificateLinks, "customTlsCertificateLinks", []string{}, "List of download links to custom TLS certificates. This is required to ensure trusted connections to instances with custom certificates.") diff --git a/resources/metadata/sonarExecuteScan.yaml b/resources/metadata/sonarExecuteScan.yaml index 3c67f5525a..e3230f13de 100644 --- a/resources/metadata/sonarExecuteScan.yaml +++ b/resources/metadata/sonarExecuteScan.yaml @@ -35,7 +35,8 @@ spec: - name: host - name: sonarServerUrl type: string - description: "The URL to the Sonar backend." + description: "The URL to the Sonar backend. + Jenkins only: The serverUrl parameter requires the [`instance`](#instance) parameter to be explicitly set to an empty string, as it will have no effect otherwise." scope: - PARAMETERS - STAGES From 7de5fdfa8af905f5ed66dcba9592121ddf702abc Mon Sep 17 00:00:00 2001 From: Jordi van Liempt <35920075+jliempt@users.noreply.github.com> Date: Fri, 17 May 2024 10:24:31 +0200 Subject: [PATCH 02/16] feat(gcpPublishEvent): Add additionalEventData param (#4928) * add additionalEventData param * fix double JSON marshalling * go generate * add logging of events and config * change logging to debug * add event log * fix CloudEvent JSON type * apply review feedback * fix log * add missing error handling --------- Co-authored-by: jliempt <> --- cmd/gcpPublishEvent.go | 23 +++++++++++++-- cmd/gcpPublishEvent_generated.go | 11 +++++++ pkg/events/events.go | 39 +++++++++++++++++++++++-- pkg/events/events_test.go | 22 ++++++++++++++ pkg/gcp/pubsub.go | 2 ++ resources/metadata/gcpPublishEvent.yaml | 5 ++++ 6 files changed, 97 insertions(+), 5 deletions(-) diff --git a/cmd/gcpPublishEvent.go b/cmd/gcpPublishEvent.go index 3ae5eb5b39..46600112af 100644 --- a/cmd/gcpPublishEvent.go +++ b/cmd/gcpPublishEvent.go @@ -81,7 +81,7 @@ func runGcpPublishEvent(utils gcpPublishEventUtils) error { log.Entry().WithError(err).Warning("Cannot infer config from CI environment") } - data, err = events.NewEvent(config.EventType, config.EventSource).CreateWithJSONData(config.EventData).ToBytes() + data, err = createNewEvent(config) if err != nil { return errors.Wrap(err, "failed to create event data") } @@ -101,7 +101,26 @@ func runGcpPublishEvent(utils gcpPublishEventUtils) error { return errors.Wrap(err, "failed to publish event") } - log.Entry().Info("event published successfully!") + log.Entry().Infof("Event published successfully! With topic: %s", config.Topic) return nil } + +func createNewEvent(config *gcpPublishEventOptions) ([]byte, error) { + event, err := events.NewEvent(config.EventType, config.EventSource).CreateWithJSONData(config.EventData) + if err != nil { + return []byte{}, errors.Wrap(err, "failed to create new event") + } + + err = event.AddToCloudEventData(config.AdditionalEventData) + if err != nil { + log.Entry().Debugf("couldn't add additionalData to cloud event data: %s", err) + } + + eventBytes, err := event.ToBytes() + if err != nil { + return []byte{}, errors.Wrap(err, "casting event to bytes failed") + } + log.Entry().Debugf("CloudEvent created: %s", string(eventBytes)) + return eventBytes, nil +} diff --git a/cmd/gcpPublishEvent_generated.go b/cmd/gcpPublishEvent_generated.go index 47fcea9b21..d7f8eb7210 100644 --- a/cmd/gcpPublishEvent_generated.go +++ b/cmd/gcpPublishEvent_generated.go @@ -26,6 +26,7 @@ type gcpPublishEventOptions struct { EventSource string `json:"eventSource,omitempty"` EventType string `json:"eventType,omitempty"` EventData string `json:"eventData,omitempty"` + AdditionalEventData string `json:"additionalEventData,omitempty"` } // GcpPublishEventCommand Publishes an event to GCP using OIDC authentication (beta) @@ -138,6 +139,7 @@ func addGcpPublishEventFlags(cmd *cobra.Command, stepConfig *gcpPublishEventOpti cmd.Flags().StringVar(&stepConfig.EventSource, "eventSource", os.Getenv("PIPER_eventSource"), "The events source as defined by CDEvents.") cmd.Flags().StringVar(&stepConfig.EventType, "eventType", os.Getenv("PIPER_eventType"), "") cmd.Flags().StringVar(&stepConfig.EventData, "eventData", os.Getenv("PIPER_eventData"), "Data to be merged with the generated data for the cloud event data field (JSON)") + cmd.Flags().StringVar(&stepConfig.AdditionalEventData, "additionalEventData", os.Getenv("PIPER_additionalEventData"), "Data (formatted as JSON string) to add to eventData. This can be used to enrich eventData that comes from the pipeline environment.") } @@ -247,6 +249,15 @@ func gcpPublishEventMetadata() config.StepData { Aliases: []config.Alias{}, Default: os.Getenv("PIPER_eventData"), }, + { + Name: "additionalEventData", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_additionalEventData"), + }, }, }, }, diff --git a/pkg/events/events.go b/pkg/events/events.go index cd8d78e03b..700337e9f4 100644 --- a/pkg/events/events.go +++ b/pkg/events/events.go @@ -30,8 +30,17 @@ func NewEvent(eventType, eventSource string) Event { } } -func (e Event) CreateWithJSONData(data string, opts ...Option) Event { - return e.Create(data, opts...) +func (e Event) CreateWithJSONData(data string, opts ...Option) (Event, error) { + // passing a string to e.cloudEvent.SetData will result in the string being marshalled, ending up with double escape characters + // therefore pass a map instead + var dataMap map[string]interface{} + if data != "" { + err := json.Unmarshal([]byte(data), &dataMap) + if err != nil { + return e, errors.Wrap(err, "eventData is an invalid JSON") + } + } + return e.Create(dataMap, opts...), nil } func (e Event) Create(data any, opts ...Option) Event { @@ -46,7 +55,6 @@ func (e Event) Create(data any, opts ...Option) Event { for _, applyOpt := range opts { applyOpt(e.cloudEvent.Context.AsV1()) } - return e } @@ -57,3 +65,28 @@ func (e Event) ToBytes() ([]byte, error) { } return data, nil } + +func (e *Event) AddToCloudEventData(additionalDataString string) error { + if additionalDataString == "" { + return nil + } + + var additionalData map[string]interface{} + err := json.Unmarshal([]byte(additionalDataString), &additionalData) + if err != nil { + errors.Wrap(err, "couldn't add additional data to cloud event") + } + + var newEventData map[string]interface{} + err = json.Unmarshal(e.cloudEvent.DataEncoded, &newEventData) + if err != nil { + errors.Wrap(err, "couldn't add additional data to cloud event") + } + + for k, v := range additionalData { + newEventData[k] = v + } + + e.cloudEvent.SetData("application/json", newEventData) + return nil +} diff --git a/pkg/events/events_test.go b/pkg/events/events_test.go index 2ef1a0a00f..87e4742ef8 100644 --- a/pkg/events/events_test.go +++ b/pkg/events/events_test.go @@ -16,4 +16,26 @@ func TestEventCreation(t *testing.T) { assert.Equal(t, mock.Anything, event.cloudEvent.Type()) assert.Equal(t, mock.Anything, event.cloudEvent.Source()) }) + + t.Run("CreateWithJSONData - no additional data", func(t *testing.T) { + // init + testData := `{"testKey":"testValue"}` + // test + event, err := NewEvent(mock.Anything, mock.Anything).CreateWithJSONData(testData) + // asserts + assert.NoError(t, err) + assert.Equal(t, string(event.cloudEvent.Data()), testData) + }) + + t.Run("CreateWithJSONData + AddToCloudEventData", func(t *testing.T) { + // init + testData := `{"testKey": "testValue"}` + additionalData := `{"additionalKey": "additionalValue"}` + // test + event, err := NewEvent(mock.Anything, mock.Anything).CreateWithJSONData(testData) + event.AddToCloudEventData(additionalData) + // asserts + assert.NoError(t, err) + assert.Equal(t, string(event.cloudEvent.Data()), `{"additionalKey":"additionalValue","testKey":"testValue"}`) + }) } diff --git a/pkg/gcp/pubsub.go b/pkg/gcp/pubsub.go index 3dd604ade3..1e52bbb23a 100644 --- a/pkg/gcp/pubsub.go +++ b/pkg/gcp/pubsub.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" + "github.com/SAP/jenkins-library/pkg/log" "github.com/pkg/errors" ) @@ -38,6 +39,7 @@ func Publish(projectNumber string, topic string, token string, key string, data if err != nil { return errors.Wrap(err, "failed to marshal event") } + log.Entry().Debugf("created pubsub event: %s", string(eventBytes)) // create request request, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf(api_url, projectNumber, topic), bytes.NewReader(eventBytes)) diff --git a/resources/metadata/gcpPublishEvent.yaml b/resources/metadata/gcpPublishEvent.yaml index fb7abd048f..4fa43826e8 100644 --- a/resources/metadata/gcpPublishEvent.yaml +++ b/resources/metadata/gcpPublishEvent.yaml @@ -81,3 +81,8 @@ spec: resourceRef: - name: commonPipelineEnvironment param: custom/eventData + - name: additionalEventData + description: Data (formatted as JSON string) to add to eventData. This can be used to enrich eventData that comes from the pipeline environment. + type: string + scope: + - PARAMETERS From bbd087e03d40cb7f3ca0f74d68b81fcb23a4965e Mon Sep 17 00:00:00 2001 From: Vyacheslav Starostin <32613074+vstarostin@users.noreply.github.com> Date: Tue, 21 May 2024 14:35:26 +0500 Subject: [PATCH 03/16] Add Vault resource to parameters (#4932) --- pkg/documentation/generator/parameters.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/documentation/generator/parameters.go b/pkg/documentation/generator/parameters.go index f52a4ba712..bb66c832b9 100644 --- a/pkg/documentation/generator/parameters.go +++ b/pkg/documentation/generator/parameters.go @@ -337,6 +337,9 @@ func resourceReferenceDetails(resourceRef []config.ResourceReference) string { func addVaultResourceDetails(resource config.ResourceReference, resourceDetails string) string { if resource.Type == "vaultSecret" || resource.Type == "vaultSecretFile" { + resourceDetails += "
Vault resource:
" + resourceDetails += fmt.Sprintf("  name: `%v`
", resource.Name) + resourceDetails += fmt.Sprintf("  default value: `%v`
", resource.Default) resourceDetails += "
Vault paths:
" resourceDetails += "