chore: do not record canary sync successes to job history (#1245)

* chore: do not record canary sync successes to job history

* chore: remove time.Sleep and add more context to error log

pkg/jobs/canary/canary_jobs.go

@@ -347,23 +347,24 @@ func SyncCanaryJobs() {
 	canaries, err := db.GetAllCanariesForSync()
 	if err != nil {
 		logger.Errorf("Failed to get canaries: %v", err)
+
+		jobHistory := models.NewJobHistory("SyncCanaries", "canary", "").Start()
+		logIfError(db.PersistJobHistory(jobHistory.AddError(err.Error()).End()), "failed to persist job history [SyncCanaries]")
+
 		return
 	}
 
 	existingIDsInCron := getAllCanaryIDsInCron()
-	var idsInNewFetch []string
+	idsInNewFetch := make([]string, 0, len(canaries))
 	for _, c := range canaries {
 		jobHistory := models.NewJobHistory("CanarySync", "canary", c.ID.String()).Start()
-		_ = db.PersistJobHistory(jobHistory)
 
 		idsInNewFetch = append(idsInNewFetch, c.ID.String())
 		if err := SyncCanaryJob(c); err != nil {
 			logger.Errorf("Error syncing canary[%s]: %v", c.ID, err.Error())
-			_ = db.PersistJobHistory(jobHistory.AddError(err.Error()).End())
+			logIfError(db.PersistJobHistory(jobHistory.AddError(err.Error()).End()), "failed to persist job history [CanarySync]")
 			continue
 		}
-		jobHistory.IncrSuccess()
-		_ = db.PersistJobHistory(jobHistory.End())
 	}
 
 	idsToRemoveFromCron := utils.SetDifference(existingIDsInCron, idsInNewFetch)
@@ -385,9 +386,12 @@ func DeleteCanaryJob(id string) {
 
 func ReconcileCanaryChecks() {
 	logger.Infof("Reconciling Canary Checks")
+
 	jobHistory := models.NewJobHistory("ReconcileCanaryChecks", "", "").Start()
-	_ = db.PersistJobHistory(jobHistory)
-	defer func() { _ = db.PersistJobHistory(jobHistory.End()) }()
+	logIfError(db.PersistJobHistory(jobHistory), "failed to persist job history [ReconcileCanaryChecks start]")
+	defer func() {
+		logIfError(db.PersistJobHistory(jobHistory.End()), "failed to persist job history [ReconcileCanaryChecks end]")
+	}()
 
 	canaries, err := db.GetAllCanariesForSync()
 	if err != nil {
@@ -448,3 +452,9 @@ func init() {
 	// We are adding a small buffer to prevent blocking
 	CanaryStatusChannel = make(chan CanaryStatusPayload, 64)
 }
+
+func logIfError(err error, description string) {
+	if err != nil {
+		logger.Errorf("%s: %v", description, err)
+	}
+}