Skip to content

fix workflow backup issue for azure devops #246

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 14, 2025
Merged

fix workflow backup issue for azure devops #246

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 114 additions & 52 deletions cloudSync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,12 @@ import (
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"

//"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/protocol/packp/capability"
"github.com/go-git/go-git/v5/plumbing/transport"
gitHttp "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/go-git/go-git/v5/storage/memory"

uuid "github.com/satori/go.uuid"
Expand Down Expand Up @@ -1112,7 +1116,12 @@ func SetGitWorkflow(ctx context.Context, workflow Workflow, org *Org) error {
}

if len(org.Defaults.WorkflowUploadBranch) == 0 {
org.Defaults.WorkflowUploadBranch = "master"
// Default to 'main' for Azure DevOps, 'master' for others
if strings.Contains(org.Defaults.WorkflowUploadRepo, "dev.azure.com") {
org.Defaults.WorkflowUploadBranch = "main"
} else {
org.Defaults.WorkflowUploadBranch = "master"
}
}

if org.Defaults.WorkflowUploadRepo == "" || org.Defaults.WorkflowUploadToken == "" {
Expand All @@ -1122,18 +1131,8 @@ func SetGitWorkflow(ctx context.Context, workflow Workflow, org *Org) error {
}

org.Defaults.WorkflowUploadRepo = strings.TrimSpace(org.Defaults.WorkflowUploadRepo)
if strings.HasPrefix(org.Defaults.WorkflowUploadRepo, "https://") {
org.Defaults.WorkflowUploadRepo = strings.Replace(org.Defaults.WorkflowUploadRepo, "https://", "", 1)
org.Defaults.WorkflowUploadRepo = strings.Replace(org.Defaults.WorkflowUploadRepo, "http://", "", 1)
}

if strings.HasSuffix(org.Defaults.WorkflowUploadRepo, ".git") {
org.Defaults.WorkflowUploadRepo = strings.TrimSuffix(org.Defaults.WorkflowUploadRepo, ".git")
}

//log.Printf("[DEBUG] Uploading workflow %s to repo %s for org %s (%s)", workflow.ID, org.Defaults.WorkflowUploadRepo, org.Name, org.Id)

// Remove images
// Remove images from workflow before backup
workflow.Image = ""
for actionIndex, _ := range workflow.Actions {
workflow.Actions[actionIndex].LargeImage = ""
Expand All @@ -1155,12 +1154,39 @@ func SetGitWorkflow(ctx context.Context, workflow Workflow, org *Org) error {
return err
}

commitMessage := fmt.Sprintf("User '%s' updated workflow '%s' with status '%s'", workflow.UpdatedBy, workflow.Name, workflow.Status)
urlEncodedPassword := url.QueryEscape(org.Defaults.WorkflowUploadToken)
location := fmt.Sprintf("https://%s:%s@%s.git", org.Defaults.WorkflowUploadUsername, urlEncodedPassword, org.Defaults.WorkflowUploadRepo)
commitMessage := fmt.Sprintf("User '%s' updated workflow '%s' with status '%s' at %s", workflow.UpdatedBy, workflow.Name, workflow.Status, time.Now().Format("2006-01-02 15:04:05"))
repoURL := org.Defaults.WorkflowUploadRepo
repoURL = strings.TrimPrefix(repoURL, "https://")
repoURL = strings.TrimPrefix(repoURL, "http://")

var location string
var isAzureDevOps bool

if strings.Contains(repoURL, "dev.azure.com") {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And this I guess

Interesting how we gotta cleanup a lot 🤔

isAzureDevOps = true
location = fmt.Sprintf("https://%s", repoURL)
log.Printf("[DEBUG] Detected Azure DevOps repository")
} else {
isAzureDevOps = false
// Only append .git if the URL contains github.com
if strings.Contains(repoURL, "github.com") && !strings.HasSuffix(repoURL, ".git") {
repoURL += ".git"
}

urlEncodedPassword := url.QueryEscape(org.Defaults.WorkflowUploadToken)
location = fmt.Sprintf("https://%s:%s@%s", org.Defaults.WorkflowUploadUsername, urlEncodedPassword, repoURL)

}

maskedRepo := location
if !isAzureDevOps {
maskedRepo = strings.ReplaceAll(location, org.Defaults.WorkflowUploadToken, "****")
if urlEncodedPassword := url.QueryEscape(org.Defaults.WorkflowUploadToken); urlEncodedPassword != org.Defaults.WorkflowUploadToken {
maskedRepo = strings.ReplaceAll(maskedRepo, urlEncodedPassword, "****")
}
}

newRepoName := strings.Replace(strings.Replace(location, org.Defaults.WorkflowUploadToken, "****", -1), urlEncodedPassword, "****", -1)
log.Printf("[DEBUG] Uploading workflow %s to repo: %s", workflow.ID, newRepoName)
log.Printf("[DEBUG] Uploading workflow %s to repo: %s", workflow.ID, maskedRepo)

fs := memfs.New()
if len(workflow.Status) == 0 {
Expand All @@ -1170,65 +1196,94 @@ func SetGitWorkflow(ctx context.Context, workflow Workflow, org *Org) error {
//filePath := fmt.Sprintf("/%s/%s.json", workflow.Status, workflow.ID)
filePath := fmt.Sprintf("%s/%s/%s_%s.json", workflow.ExecutingOrg.Id, workflow.Status, strings.ReplaceAll(workflow.Name, " ", "-"), workflow.ID)

// Specify the file path within the repository
repo, err := git.Clone(memory.NewStorage(), fs, &git.CloneOptions{
cloneOptions := &git.CloneOptions{
URL: location,
})
if err != nil {
newErr := strings.ReplaceAll(err.Error(), org.Defaults.WorkflowUploadToken, "****")
newErr = strings.ReplaceAll(newErr, urlEncodedPassword, "****")

log.Printf("[ERROR] Error cloning repo '%s' (workflow backup): %s", newRepoName, newErr)
return err
}

// Initialize a new Git repository in memory
w := &git.Worktree{}
// For Azure DevOps, add additional auth configuration and capability handling
if isAzureDevOps {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this it? I'm so confused why so much code was changed D:

transport.UnsupportedCapabilities = []capability.Capability{
capability.ThinPack,
}

cloneOptions.Auth = &gitHttp.BasicAuth{
Username: org.Defaults.WorkflowUploadUsername, // Use the username for Azure DevOps
Password: org.Defaults.WorkflowUploadToken,
}
}

// Create a new commit with the in-memory file
w, err = repo.Worktree()
repo, err := git.Clone(memory.NewStorage(), fs, cloneOptions)
if err != nil {
newErr := strings.ReplaceAll(err.Error(), org.Defaults.WorkflowUploadToken, "****")
newErr = strings.ReplaceAll(newErr, urlEncodedPassword, "****")
errMsg := err.Error()
if isAzureDevOps {
log.Printf("[ERROR] Azure DevOps clone failed. Check: 1) PAT has Code(R/W) permissions, 2) Branch '%s' exists, 3) Repo URL is correct", org.Defaults.WorkflowUploadBranch)
}
log.Printf("[ERROR] Error cloning repo '%s': %s", maskedRepo, strings.ReplaceAll(errMsg, org.Defaults.WorkflowUploadToken, "****"))
return err
}

log.Printf("[ERROR] Error getting worktree for repo '%s' (2): %s", newRepoName, newErr)
w, err := repo.Worktree()
if err != nil {
log.Printf("[ERROR] Error getting worktree for repo '%s': %s", maskedRepo, err)
return err
}

// Write the byte blob to the in-memory file system
file, err := fs.Create(filePath)
if err != nil {
newErr := strings.ReplaceAll(err.Error(), org.Defaults.WorkflowUploadToken, "****")
log.Printf("[ERROR] Creating file in repo '%s': %v", maskedRepo, err)
return err
}
defer file.Close()

log.Printf("[ERROR] Creating git file: %v", newErr)
if _, err = io.Copy(file, bytes.NewReader(workflowData)); err != nil {
log.Printf("[ERROR] Writing data to file: %v", err)
return err
}

defer file.Close()
//_, err = io.Copy(file, bytes.NewReader(workflowData))
_, err = io.Copy(file, bytes.NewReader(workflowData))
if err != nil {
log.Printf("[ERROR] Writing data to git file: %v", err)
if _, err = w.Add(filePath); err != nil {
log.Printf("[ERROR] Adding file to staging area: %s", err)
return err
}

// Add the file to the staging area
_, err = w.Add(filePath)
// Check if there are any changes to commit
status, err := w.Status()
if err != nil {
log.Printf("[ERROR] Error adding file to git staging area (2): %s", err)
log.Printf("[ERROR] Getting working tree status: %v", err)
return err
}

// Commit the changes
hasChanges := false
for _, fileStatus := range status {
if fileStatus.Staging != git.Unmodified {
hasChanges = true
break
}
}

if !hasChanges {
log.Printf("[INFO] No changes detected for workflow %s (%s). File content is identical to existing version.", workflow.Name, workflow.ID)
return nil
}

authorName := org.Defaults.WorkflowUploadUsername
if authorName == "" {
if isAzureDevOps {
authorName = "Workflow Automation"
} else {
authorName = "Shuffle User"
}
}

_, err = w.Commit(commitMessage, &git.CommitOptions{
Author: &object.Signature{
Name: org.Defaults.WorkflowUploadUsername,
Name: authorName,
Email: "",
When: time.Now(),
},
})
if err != nil {
log.Printf("[ERROR] Committing git changes: %v (2)", err)
log.Printf("[ERROR] Committing changes: %v", err)
return err
}

Expand All @@ -1237,18 +1292,25 @@ func SetGitWorkflow(ctx context.Context, workflow Workflow, org *Org) error {
// Push the changes to a remote repository (replace URL with your repository URL)
// fmt.Sprintf("refs/heads/%s:refs/heads/%s", org.Defaults.WorkflowUploadBranch, org.Defaults.WorkflowUploadBranch)},
ref := fmt.Sprintf("refs/heads/%s:refs/heads/%s", org.Defaults.WorkflowUploadBranch, org.Defaults.WorkflowUploadBranch)
err = repo.Push(&git.PushOptions{

pushOptions := &git.PushOptions{
RemoteName: "origin",
RefSpecs: []config.RefSpec{config.RefSpec(ref)},
RemoteURL: location,
})
}

// Set authentication for push operations for both Azure DevOps and GitHub
pushOptions.Auth = &gitHttp.BasicAuth{
Username: org.Defaults.WorkflowUploadUsername,
Password: org.Defaults.WorkflowUploadToken,
}

err = repo.Push(pushOptions)
if err != nil {
log.Printf("[ERROR] Change git Push issue: %v (2)", err)
log.Printf("[ERROR] Git push failed for repo '%s': %v", maskedRepo, err)
return err
}

log.Printf("[DEBUG] File uploaded successfully to '%s'!", newRepoName)

log.Printf("[DEBUG] Workflow successfully uploaded to '%s'!", maskedRepo)
return nil
}

Expand Down
Loading