diff --git a/activator/activator.go b/activator/activator.go index 47dbf732..f2365440 100644 --- a/activator/activator.go +++ b/activator/activator.go @@ -3,6 +3,8 @@ package activator type ActivatedStep struct { StepYMLPath string + ExecutablePath string + // DidStepLibUpdate indicates that the local steplib cache was updated while resolving the exact step version. DidStepLibUpdate bool } diff --git a/activator/steplib/activate.go b/activator/steplib/activate.go index 0f192a5d..e3a0d15c 100644 --- a/activator/steplib/activate.go +++ b/activator/steplib/activate.go @@ -4,7 +4,9 @@ import ( "fmt" "os" "path/filepath" + "runtime" "slices" + "time" "github.com/bitrise-io/go-utils/command" "github.com/bitrise-io/go-utils/pathutil" @@ -12,49 +14,45 @@ import ( "github.com/bitrise-io/stepman/stepman" ) -var errStepNotAvailableOfflineMode error = fmt.Errorf("step not available in offline mode") +const precompiledStepsEnv = "BITRISE_EXPERIMENT_PRECOMPILED_STEPS" -func ActivateStep(stepLibURI, id, version, destination, destinationStepYML string, log stepman.Logger, isOfflineMode bool) error { +func ActivateStep(stepLibURI, id, version, destination, destinationStepYML string, log stepman.Logger, isOfflineMode bool) (string, error) { stepCollection, err := stepman.ReadStepSpec(stepLibURI) if err != nil { - return fmt.Errorf("failed to read %s steplib: %s", stepLibURI, err) + return "", fmt.Errorf("failed to read %s steplib: %s", stepLibURI, err) } - step, version, err := queryStep(stepCollection, stepLibURI, id, version) + step, version, err := queryStepMetadata(stepCollection, stepLibURI, id, version) if err != nil { - return fmt.Errorf("failed to find step: %s", err) - } - - srcFolder, err := activateStep(stepCollection, stepLibURI, id, version, step, log, isOfflineMode) - if err != nil { - if err == errStepNotAvailableOfflineMode { - availableVersions := ListCachedStepVersions(log, stepCollection, stepLibURI, id) - versionList := "Other versions available in the local cache:" - for _, version := range availableVersions { - versionList = versionList + fmt.Sprintf("\n- %s", version) + return "", fmt.Errorf("failed to find step: %s", err) + } + + if os.Getenv(precompiledStepsEnv) == "true" { + platform := fmt.Sprintf("%s-%s", runtime.GOOS, runtime.GOARCH) + executableForPlatform, ok := step.Executables[platform] + if ok { + log.Debugf("Downloading executable for %s", platform) + downloadStart := time.Now() + execPath, err := activateStepExecutable(stepLibURI, id, version, executableForPlatform, destination, destinationStepYML) + if err != nil { + log.Warnf("Failed to download step executable, falling back to source build: %s", err) + err = activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode) + return "", err } - - errMsg := fmt.Sprintf("version is not available in the local cache and $BITRISE_OFFLINE_MODE is set. %s", versionList) - return fmt.Errorf("failed to download step: %s", errMsg) - } - - return fmt.Errorf("failed to download step: %s", err) - } - - if err := copyStep(srcFolder, destination); err != nil { - return fmt.Errorf("copy step failed: %s", err) - } - - if destinationStepYML != "" { - if err := copyStepYML(stepLibURI, id, version, destinationStepYML); err != nil { - return fmt.Errorf("copy step.yml failed: %s", err) + log.Debugf("Downloaded executable in %s", time.Since(downloadStart).Round(time.Millisecond)) + return execPath, nil + } else { + log.Infof("No prebuilt executable found for %s, falling back to source build", platform) + err = activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode) + return "", err } + } else { + err = activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode) + return "", err } - - return nil } -func queryStep(stepLib models.StepCollectionModel, stepLibURI string, id, version string) (models.StepModel, string, error) { +func queryStepMetadata(stepLib models.StepCollectionModel, stepLibURI string, id, version string) (models.StepModel, string, error) { step, stepFound, versionFound := stepLib.GetStep(id, version) if !stepFound { @@ -75,46 +73,6 @@ func queryStep(stepLib models.StepCollectionModel, stepLibURI string, id, versio return step, version, nil } -func activateStep(stepLib models.StepCollectionModel, stepLibURI, id, version string, step models.StepModel, log stepman.Logger, isOfflineMode bool) (string, error) { - route, found := stepman.ReadRoute(stepLibURI) - if !found { - return "", fmt.Errorf("no route found for %s steplib", stepLibURI) - } - - stepCacheDir := stepman.GetStepCacheDirPath(route, id, version) - if exist, err := pathutil.IsPathExists(stepCacheDir); err != nil { - return "", fmt.Errorf("failed to check if %s path exist: %s", stepCacheDir, err) - } else if exist { - return stepCacheDir, nil - } - - // version specific source cache not exists - if isOfflineMode { - return "", errStepNotAvailableOfflineMode - } - - if err := stepman.DownloadStep(stepLibURI, stepLib, id, version, step.Source.Commit, log); err != nil { - return "", fmt.Errorf("download failed: %s", err) - } - - return stepCacheDir, nil -} - -func copyStep(src, dst string) error { - if exist, err := pathutil.IsPathExists(dst); err != nil { - return fmt.Errorf("failed to check if %s path exist: %s", dst, err) - } else if !exist { - if err := os.MkdirAll(dst, 0777); err != nil { - return fmt.Errorf("failed to create dir for %s path: %s", dst, err) - } - } - - if err := command.CopyDir(src+"/", dst, true); err != nil { - return fmt.Errorf("copy command failed: %s", err) - } - return nil -} - func copyStepYML(libraryURL, id, version, dest string) error { route, found := stepman.ReadRoute(libraryURL) if !found { @@ -138,8 +96,14 @@ func copyStepYML(libraryURL, id, version, dest string) error { func ListCachedStepVersions(log stepman.Logger, stepLib models.StepCollectionModel, stepLibURI, stepID string) []string { versions := []models.Semver{} - for version, step := range stepLib.Steps[stepID].Versions { - _, err := activateStep(stepLib, stepLibURI, stepID, version, step, log, true) + route, found := stepman.ReadRoute(stepLibURI) + if !found { + return nil + } + + for version := range stepLib.Steps[stepID].Versions { + stepCacheDir := stepman.GetStepCacheDirPath(route, stepID, version) + _, err := os.Stat(stepCacheDir) if err != nil { continue } diff --git a/activator/steplib/activate_executable.go b/activator/steplib/activate_executable.go new file mode 100644 index 00000000..5b2b9507 --- /dev/null +++ b/activator/steplib/activate_executable.go @@ -0,0 +1,89 @@ +package steplib + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/bitrise-io/stepman/models" + "github.com/hashicorp/go-retryablehttp" +) + +func activateStepExecutable( + stepLibURI string, + stepID string, + version string, + executable models.Executable, + destination string, + destinationStepYML string, +) (string, error) { + resp, err := retryablehttp.Get(executable.Url) + if err != nil { + return "", fmt.Errorf("fetch from %s: %w", executable.Url, err) + } + defer resp.Body.Close() + + err = os.MkdirAll(destination, 0755) + if err != nil { + return "", fmt.Errorf("create directory %s: %w", destination, err) + } + + path := filepath.Join(destination, stepID) + file, err := os.Create(path) + if err != nil { + return "", fmt.Errorf("create file %s: %w", path, err) + } + + _, err = io.Copy(file, resp.Body) + if err != nil { + return "", fmt.Errorf("download %s to %s: %w", executable.Url, path, err) + } + + err = validateHash(path, executable.Hash) + if err != nil { + return "", fmt.Errorf("validate hash: %s", err) + } + + err = os.Chmod(path, 0755) + if err != nil { + return "", fmt.Errorf("set executable permission on file: %s", err) + } + + if err := copyStepYML(stepLibURI, stepID, version, destinationStepYML); err != nil { + return "", fmt.Errorf("copy step.yml: %s", err) + } + + return path, nil +} + +func validateHash(filePath string, expectedHash string) error { + if expectedHash == "" { + return fmt.Errorf("hash is empty") + } + + if !strings.HasPrefix(expectedHash, "sha256-") { + return fmt.Errorf("only SHA256 hashes supported at this time, make sure to prefix the hash with `sha256-`. Found hash value: %s", expectedHash) + } + + expectedHash = strings.TrimPrefix(expectedHash, "sha256-") + + reader, err := os.Open(filePath) + if err != nil { + return err + } + + h := sha256.New() + _, err = io.Copy(h, reader) + if err != nil { + return fmt.Errorf("calculate hash: %w", err) + } + actualHash := hex.EncodeToString(h.Sum(nil)) + if actualHash != expectedHash { + return fmt.Errorf("hash mismatch: expected sha256-%s, got sha256-%s", expectedHash, actualHash) + } + return nil +} diff --git a/activator/steplib/activate_executable_test.go b/activator/steplib/activate_executable_test.go new file mode 100644 index 00000000..a244b213 --- /dev/null +++ b/activator/steplib/activate_executable_test.go @@ -0,0 +1,60 @@ +package steplib + +import ( + "fmt" + "testing" +) + + +func TestValidateHash(t *testing.T) { + tests := []struct { + name string + filePath string + expectedHash string + expectedErr error + }{ + { + name: "Valid hash", + filePath: "testdata/file.txt", + expectedHash: "sha256-f2040af3939f5033be8ca9b363055b3e53107c4688ba39b71d4529869a9cc9b2", + expectedErr: nil, + }, + { + name: "Hash mismatch", + filePath: "testdata/file.txt", + expectedHash: "sha256-1234567890abcdef", + expectedErr: fmt.Errorf("hash mismatch: expected sha256-1234567890abcdef, got sha256-f2040af3939f5033be8ca9b363055b3e53107c4688ba39b71d4529869a9cc9b2"), + }, + { + name: "Nonexistent file", + filePath: "testdata/nonexistent.txt", + expectedHash: "sha256-3b6b4f1e2e8b8a9e4f7a4b5e6c7d8e9f", + expectedErr: fmt.Errorf("open testdata/nonexistent.txt: no such file or directory"), + }, + { + name: "Empty hash", + filePath: "testdata/file.txt", + expectedHash: "", + expectedErr: fmt.Errorf("hash is empty"), + }, + { + name: "Invalid hash type", + filePath: "testdata/file.txt", + expectedHash: "md5-3b6b4f1e2e8b8a9e4f7a4b5e6c7d8e9f", + expectedErr: fmt.Errorf("only SHA256 hashes supported at this time, make sure to prefix the hash with `sha256-`. Found hash value: md5-3b6b4f1e2e8b8a9e4f7a4b5e6c7d8e9f"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateHash(tt.filePath, tt.expectedHash) + if err != nil && tt.expectedErr == nil { + t.Errorf("unexpected error: %s", err) + } else if err == nil && tt.expectedErr != nil { + t.Errorf("expected error: %s, but got nil", tt.expectedErr) + } else if err != nil && tt.expectedErr != nil && err.Error() != tt.expectedErr.Error() { + t.Errorf("expected error: %s, but got: %s", tt.expectedErr, err) + } + }) + } +} diff --git a/activator/steplib/activate_source.go b/activator/steplib/activate_source.go new file mode 100644 index 00000000..38975aea --- /dev/null +++ b/activator/steplib/activate_source.go @@ -0,0 +1,77 @@ +package steplib + +import ( + "fmt" + "os" + + "github.com/bitrise-io/go-utils/command" + "github.com/bitrise-io/go-utils/pathutil" + "github.com/bitrise-io/stepman/models" + "github.com/bitrise-io/stepman/stepman" +) + +func activateStepSource( + stepLib models.StepCollectionModel, + stepLibURI, id, version string, + step models.StepModel, + destination string, + stepYMLDestination string, + log stepman.Logger, + isOfflineMode bool, +) error { + route, found := stepman.ReadRoute(stepLibURI) + if !found { + return fmt.Errorf("no route found for %s steplib", stepLibURI) + } + + stepCacheDir := stepman.GetStepCacheDirPath(route, id, version) + if exist, err := pathutil.IsPathExists(stepCacheDir); err != nil { + return fmt.Errorf("failed to check if %s path exist: %s", stepCacheDir, err) + } else if exist { + if err := copyStep(stepCacheDir, destination); err != nil { + return fmt.Errorf("copy step failed: %s", err) + } + return nil + } + + // version specific source cache not exists + if isOfflineMode { + availableVersions := ListCachedStepVersions(log, stepLib, stepLibURI, id) + versionList := "Other versions available in the local cache:" + for _, version := range availableVersions { + versionList = versionList + fmt.Sprintf("\n- %s", version) + } + + errMsg := fmt.Sprintf("version is not available in the local cache and $BITRISE_OFFLINE_MODE is set. %s", versionList) + return fmt.Errorf("download step: %s", errMsg) + } + + if err := stepman.DownloadStep(stepLibURI, stepLib, id, version, step.Source.Commit, log); err != nil { + return fmt.Errorf("download failed: %s", err) + } + + if err := copyStep(stepCacheDir, destination); err != nil { + return fmt.Errorf("copy step failed: %s", err) + } + + if err := copyStepYML(stepLibURI, id, version, stepYMLDestination); err != nil { + return fmt.Errorf("copy step.yml failed: %s", err) + } + + return nil +} + +func copyStep(src, dst string) error { + if exist, err := pathutil.IsPathExists(dst); err != nil { + return fmt.Errorf("failed to check if %s path exist: %s", dst, err) + } else if !exist { + if err := os.MkdirAll(dst, 0777); err != nil { + return fmt.Errorf("failed to create dir for %s path: %s", dst, err) + } + } + + if err := command.CopyDir(src+"/", dst, true); err != nil { + return fmt.Errorf("copy command failed: %s", err) + } + return nil +} diff --git a/activator/steplib/testdata/file.txt b/activator/steplib/testdata/file.txt new file mode 100644 index 00000000..4335f02a --- /dev/null +++ b/activator/steplib/testdata/file.txt @@ -0,0 +1 @@ +hash verification test diff --git a/activator/steplib_ref.go b/activator/steplib_ref.go index 30e1efda..b18d84e3 100644 --- a/activator/steplib_ref.go +++ b/activator/steplib_ref.go @@ -33,10 +33,11 @@ func ActivateSteplibRefStep( return activationResult, err } - err = steplib.ActivateStep(id.SteplibSource, id.IDorURI, stepInfo.Version, activatedStepDir, stepYMLPath, log, isOfflineMode) + execPath, err := steplib.ActivateStep(id.SteplibSource, id.IDorURI, stepInfo.Version, activatedStepDir, stepYMLPath, log, isOfflineMode) if err != nil { return activationResult, err } + activationResult.ExecutablePath = execPath // TODO: this is sketchy, we should clean this up, but this pointer originates in the CLI codebase stepInfoPtr.ID = stepInfo.ID diff --git a/models/models.go b/models/models.go index ec6a31c5..98d81be8 100644 --- a/models/models.go +++ b/models/models.go @@ -56,12 +56,11 @@ type SwiftStepToolkitModel struct { } type StepToolkitModel struct { - Bash *BashStepToolkitModel `json:"bash,omitempty" yaml:"bash,omitempty"` - Go *GoStepToolkitModel `json:"go,omitempty" yaml:"go,omitempty"` - Swift *SwiftStepToolkitModel `json:"swift,omitempty" yaml:"swift,omitempty"` + Bash *BashStepToolkitModel `json:"bash,omitempty" yaml:"bash,omitempty"` + Go *GoStepToolkitModel `json:"go,omitempty" yaml:"go,omitempty"` + Swift *SwiftStepToolkitModel `json:"swift,omitempty" yaml:"swift,omitempty"` } -// StepModel ... type StepModel struct { Title *string `json:"title,omitempty" yaml:"title,omitempty"` Summary *string `json:"summary,omitempty" yaml:"summary,omitempty"` @@ -70,10 +69,13 @@ type StepModel struct { Website *string `json:"website,omitempty" yaml:"website,omitempty"` SourceCodeURL *string `json:"source_code_url,omitempty" yaml:"source_code_url,omitempty"` SupportURL *string `json:"support_url,omitempty" yaml:"support_url,omitempty"` + // auto-generated at share PublishedAt *time.Time `json:"published_at,omitempty" yaml:"published_at,omitempty"` Source *StepSourceModel `json:"source,omitempty" yaml:"source,omitempty"` + Executables Executables `json:"executables,omitempty" yaml:"executables,omitempty"` AssetURLs map[string]string `json:"asset_urls,omitempty" yaml:"asset_urls,omitempty"` + // HostOsTags []string `json:"host_os_tags,omitempty" yaml:"host_os_tags,omitempty"` ProjectTypeTags []string `json:"project_type_tags,omitempty" yaml:"project_type_tags,omitempty"` @@ -120,6 +122,15 @@ type StepGroupModel struct { Versions map[string]StepModel `json:"versions,omitempty" yaml:"versions,omitempty"` } +// Key: platform, as in runtime.GOOS + runtime.GOARCH +// Examples: darwin-arm64, linux-amd64 +type Executables map[string]Executable + +type Executable struct { + Url string `json:"url,omitempty" yaml:"url,omitempty"` + Hash string `json:"hash,omitempty" yaml:"hash,omitempty"` +} + func (stepGroup StepGroupModel) LatestVersion() (StepModel, bool) { step, found := stepGroup.Versions[stepGroup.LatestVersionNumber] if !found { @@ -174,4 +185,3 @@ type SteplibInfoModel struct { URI string `json:"uri,omitempty" yaml:"uri,omitempty"` SpecPath string `json:"spec_path,omitempty" yaml:"spec_path,omitempty"` } - diff --git a/toolkits/golang_test.go b/toolkits/golang_test.go index b5a99313..d1999ca5 100644 --- a/toolkits/golang_test.go +++ b/toolkits/golang_test.go @@ -219,7 +219,7 @@ func Benchmark_goBuildStep(b *testing.B) { require.NoError(b, err) }() - err = steplib.ActivateStep("https://github.com/bitrise-io/bitrise-steplib", "xcode-test", "5.1.1", stepDir, "", logger, false) + _, err = steplib.ActivateStep("https://github.com/bitrise-io/bitrise-steplib", "xcode-test", "5.1.1", stepDir, "", logger, false) require.NoError(b, err) packageName := "github.com/bitrise-steplib/steps-xcode-test"