From 1bb64b8aeffab532d1ac261c7aa7920500ec8fe0 Mon Sep 17 00:00:00 2001 From: Grant Arnold Date: Mon, 4 Mar 2024 16:34:26 +0000 Subject: [PATCH 1/5] Start implementation of projects v4 --- cmd/projects_upload.go | 110 +++++++++++++++++++++++++---------------- 1 file changed, 67 insertions(+), 43 deletions(-) diff --git a/cmd/projects_upload.go b/cmd/projects_upload.go index 080dff8..bef3aa3 100644 --- a/cmd/projects_upload.go +++ b/cmd/projects_upload.go @@ -10,6 +10,7 @@ import ( "strconv" "github.com/spf13/cobra" + "github.com/spf13/viper" ) type projectUploadFlags struct { @@ -18,12 +19,15 @@ type projectUploadFlags struct { pauseNotifications bool projectsImportMode string disableProjectItems bool + apiVersion apiVersionFlag } type ProjectUploadTask struct { Id int `json:"id"` } +var projectUploadV4BuildThreshold = 23766 + func newProjectUploadCmd() *cobra.Command { f := projectUploadFlags{} cmd := &cobra.Command{ @@ -58,6 +62,9 @@ func newProjectUploadCmd() *cobra.Command { cmd.Flags().BoolVar(&f.pauseNotifications, "pause-notifications", true, "Disable notifications for the duration of the restore.") cmd.Flags().StringVar(&f.projectsImportMode, "projects-import-mode", "", "Import mode for projects. To import only projects in the import package that do not exist on the current instance, specify INSERT. To overwrite projects on the current instance with those in the import package, specify UPDATE. If not supplied, importMode will be used.") cmd.Flags().BoolVar(&f.disableProjectItems, "disable-project-items", false, "Whether to disable items in the imported FME Server Projects. If true, items that are new or overwritten will be imported but disabled. If false, project items are imported as defined in the import package.") + cmd.Flags().Var(&f.apiVersion, "api-version", "The api version to use when contacting FME Server. Must be one of v3 or v4") + cmd.Flags().MarkHidden("api-version") + cmd.RegisterFlagCompletionFunc("api-version", apiVersionFlagCompletion) cmd.MarkFlagRequired("file") return cmd @@ -66,6 +73,17 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str return func(cmd *cobra.Command, args []string) error { client := &http.Client{} + // get build to decide if we should use v3 or v4 + // FME Server 2022.0 and later can use v4. Otherwise fall back to v3 + if f.apiVersion == "" { + fmeflowBuild := viper.GetInt("build") + if fmeflowBuild < projectUploadV4BuildThreshold { + f.apiVersion = apiVersionFlagV3 + } else { + f.apiVersion = apiVersionFlagV4 + } + } + url := "" var request http.Request file, err := os.Open(f.file) @@ -74,62 +92,68 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str } defer file.Close() - url = "/fmerest/v3/projects/import/upload" - request, err = buildFmeFlowRequest(url, "POST", file) - if err != nil { - return err - } - request.Header.Set("Content-Type", "application/octet-stream") + if f.apiVersion == "v4" { - q := request.URL.Query() + } else if f.apiVersion == "v3" { - if f.pauseNotifications { - q.Add("pauseNotifications", strconv.FormatBool(f.pauseNotifications)) - } + url = "/fmerest/v3/projects/import/upload" + request, err = buildFmeFlowRequest(url, "POST", file) + if err != nil { + return err + } + request.Header.Set("Content-Type", "application/octet-stream") - if f.importMode != "" { - q.Add("importMode", f.importMode) - } + q := request.URL.Query() - if f.projectsImportMode != "" { - q.Add("projectsImportMode", f.projectsImportMode) - } + if f.pauseNotifications { + q.Add("pauseNotifications", strconv.FormatBool(f.pauseNotifications)) + } - if f.disableProjectItems { - q.Add("disableProjectItems", strconv.FormatBool(f.disableProjectItems)) - } + if f.importMode != "" { + q.Add("importMode", f.importMode) + } - request.URL.RawQuery = q.Encode() + if f.projectsImportMode != "" { + q.Add("projectsImportMode", f.projectsImportMode) + } - response, err := client.Do(&request) - if err != nil { - return err - } else if response.StatusCode != http.StatusOK { - if response.StatusCode == http.StatusInternalServerError { - return fmt.Errorf("%w: check that the file specified is a valid project file", errors.New(response.Status)) - } else { - return errors.New(response.Status) + if f.disableProjectItems { + q.Add("disableProjectItems", strconv.FormatBool(f.disableProjectItems)) } - } - responseData, err := io.ReadAll(response.Body) - if err != nil { - return err - } + request.URL.RawQuery = q.Encode() - var result ProjectUploadTask - if err := json.Unmarshal(responseData, &result); err != nil { - return err - } else { - if !jsonOutput { - fmt.Fprintln(cmd.OutOrStdout(), "Project Upload task submitted with id: "+strconv.Itoa(result.Id)) + response, err := client.Do(&request) + if err != nil { + return err + } else if response.StatusCode != http.StatusOK { + if response.StatusCode == http.StatusInternalServerError { + return fmt.Errorf("%w: check that the file specified is a valid project file", errors.New(response.Status)) + } else { + return errors.New(response.Status) + } + } + + responseData, err := io.ReadAll(response.Body) + if err != nil { + return err + } + + var result ProjectUploadTask + if err := json.Unmarshal(responseData, &result); err != nil { + return err } else { - prettyJSON, err := prettyPrintJSON(responseData) - if err != nil { - return err + if !jsonOutput { + fmt.Fprintln(cmd.OutOrStdout(), "Project Upload task submitted with id: "+strconv.Itoa(result.Id)) + } else { + prettyJSON, err := prettyPrintJSON(responseData) + if err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), prettyJSON) } - fmt.Fprintln(cmd.OutOrStdout(), prettyJSON) } + } return nil From 3b34114e656094df443f6f96de43ce2135d7b5b9 Mon Sep 17 00:00:00 2001 From: Grant Arnold Date: Fri, 8 Mar 2024 16:37:30 +0000 Subject: [PATCH 2/5] Implement v4 for project upload. Start writing tests. --- cmd/projects_download.go | 2 +- cmd/projects_download_test.go | 4 +- cmd/projects_upload.go | 516 ++++++++++++++++++++++++++++++++-- cmd/projects_upload_test.go | 82 ++++-- 4 files changed, 569 insertions(+), 35 deletions(-) diff --git a/cmd/projects_download.go b/cmd/projects_download.go index abd83bc..eea0c60 100644 --- a/cmd/projects_download.go +++ b/cmd/projects_download.go @@ -102,7 +102,7 @@ func projectDownloadRun(f *projectsDownloadFlags) func(cmd *cobra.Command, args return err } - fmt.Fprintln(cmd.OutOrStdout(), "FME Server backed up to "+f.file) + fmt.Fprintln(cmd.OutOrStdout(), "Project exported to "+f.file) return nil } } diff --git a/cmd/projects_download_test.go b/cmd/projects_download_test.go index f21704b..a66663d 100644 --- a/cmd/projects_download_test.go +++ b/cmd/projects_download_test.go @@ -46,7 +46,7 @@ func TestProjectDownload(t *testing.T) { statusCode: http.StatusOK, args: []string{"projects", "download", "--name", "TestProject", "--file", f.Name()}, body: okResponseV3, - wantOutputRegex: "FME Server backed up to", + wantOutputRegex: "Project exported to", wantFileContents: fileContents{file: f.Name(), contents: okResponseV3}, }, { @@ -54,7 +54,7 @@ func TestProjectDownload(t *testing.T) { statusCode: http.StatusOK, args: []string{"projects", "download", "--name", "TestProject", "--file", f.Name(), "--exclude-sensitive-info"}, body: okResponseV3, - wantOutputRegex: "FME Server backed up to", + wantOutputRegex: "Project exported to", wantFileContents: fileContents{file: f.Name(), contents: okResponseV3}, wantFormParams: map[string]string{"excludeSensitiveInfo": "true"}, }, diff --git a/cmd/projects_upload.go b/cmd/projects_upload.go index bef3aa3..9f06c2c 100644 --- a/cmd/projects_upload.go +++ b/cmd/projects_upload.go @@ -1,24 +1,38 @@ package cmd import ( + "bytes" "encoding/json" "errors" "fmt" "io" + "mime/multipart" "net/http" "os" "strconv" + "strings" + "time" + "github.com/AlecAivazis/survey/v2" + "github.com/jedib0t/go-pretty/v6/table" "github.com/spf13/cobra" "github.com/spf13/viper" ) type projectUploadFlags struct { file string + overwrite bool importMode string pauseNotifications bool projectsImportMode string disableProjectItems bool + getSelectable bool + selectedItems string + interactive bool + quick bool + wait bool + backupFailureTopic string + backupSuccessTopic string apiVersion apiVersionFlag } @@ -26,6 +40,60 @@ type ProjectUploadTask struct { Id int `json:"id"` } +type ProjectItems struct { + Items []ProjectItemV4 `json:"items"` + TotalCount int `json:"totalCount"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +type ProjectItemV4 struct { + ID string `json:"id"` + JobID int `json:"jobId"` + Name string `json:"name"` + Type string `json:"type"` + OwnerID string `json:"ownerId"` + OwnerName string `json:"ownerName"` + OwnerStatus string `json:"ownerStatus"` + OriginalOwner string `json:"originalOwner"` + Selected bool `json:"selected"` + Existing bool `json:"existing"` + PreviewAction string `json:"previewAction"` + Action string `json:"action"` + Source string `json:"source"` +} + +type ProjectImportRun struct { + FallbackOwnerID string `json:"fallbackOwnerID,omitempty"` + Overwrite bool `json:"overwrite"` + PauseNotifications bool `json:"pauseNotifications"` + DisableItems bool `json:"disableItems"` + Notification *ProjectNotification `json:"notification,omitempty"` + SelectedItems []ProjectSelectedItems `json:"selectedItems"` +} + +type ProjectNotification struct { + Type string `json:"type,omitempty"` + SuccessTopic string `json:"successTopic,omitempty"` + FailureTopic string `json:"failureTopic,omitempty"` +} + +type ProjectSelectedItems struct { + Type string `json:"type"` + ID string `json:"id"` +} + +type ProjectUploadV4 struct { + JobID int `json:"jobId"` + Status string `json:"status"` + Owner string `json:"owner"` + OwnerID string `json:"ownerID"` + Requested time.Time `json:"requested"` + Generated time.Time `json:"generated"` + FileName string `json:"fileName"` + Request interface{} `json:"request"` +} + var projectUploadV4BuildThreshold = 23766 func newProjectUploadCmd() *cobra.Command { @@ -35,8 +103,19 @@ func newProjectUploadCmd() *cobra.Command { Short: "Imports FME Server Projects from a downloaded package.", Long: "Imports FME Server Projects from a downloaded package. Useful for moving a project from one FME Server to another.", PreRunE: func(cmd *cobra.Command, args []string) error { + // get build to decide if we should use v3 or v4 + // FME Server 2022.0 and later can use v4. Otherwise fall back to v3 + if f.apiVersion == "" { + fmeflowBuild := viper.GetInt("build") + if fmeflowBuild < projectUploadV4BuildThreshold { + f.apiVersion = apiVersionFlagV3 + } else { + f.apiVersion = apiVersionFlagV4 + } + } + // verify import mode is valid - if f.importMode != "UPDATE" && f.importMode != "INSERT" { + if f.importMode != "UPDATE" && f.importMode != "INSERT" && f.importMode != "" { return errors.New("invalid import-mode. Must be either UPDATE or INSERT") } @@ -45,25 +124,66 @@ func newProjectUploadCmd() *cobra.Command { return errors.New("invalid projects-import-mode. Must be either UPDATE or INSERT") } + if f.apiVersion == apiVersionFlagV4 { + if f.importMode != "" { + return errors.New("cannot set the importMode flag when using the V4 API") + } + if f.projectsImportMode != "" { + return errors.New("cannot set the projectsImportMode flag when using the V3 API") + } + } + + if f.apiVersion == apiVersionFlagV3 { + if f.importMode == "" && f.projectsImportMode == "" { + if f.overwrite { + f.importMode = "UPDATE" + } else { + f.importMode = "INSERT" + } + } + } + return nil }, Example: ` # Restore from a backup in a local file fmeflow projects upload --file ProjectPackage.fsproject - # Restore from a backup in a local file using UPDATE mode - fmeflow projects upload --file ProjectPackage.fsproject --import-mode UPDATE`, + # Restore from a backup in a local file without overwriting + fmeflow projects upload --file ProjectPackage.fsproject --overwrite=false`, Args: NoArgs, RunE: projectUploadRun(&f), } cmd.Flags().StringVarP(&f.file, "file", "f", "", "Path to backup file to upload to restore. Can be a local file or the relative path inside the specified shared resource.") - cmd.Flags().StringVar(&f.importMode, "import-mode", "INSERT", "To import only items in the import package that do not exist on the current instance, specify INSERT. To overwrite items on the current instance with those in the import package, specify UPDATE. Default is INSERT.") + cmd.Flags().BoolVar(&f.overwrite, "overwrite", true, "If specified, the items in the project will overwrite existing items.") cmd.Flags().BoolVar(&f.pauseNotifications, "pause-notifications", true, "Disable notifications for the duration of the restore.") - cmd.Flags().StringVar(&f.projectsImportMode, "projects-import-mode", "", "Import mode for projects. To import only projects in the import package that do not exist on the current instance, specify INSERT. To overwrite projects on the current instance with those in the import package, specify UPDATE. If not supplied, importMode will be used.") cmd.Flags().BoolVar(&f.disableProjectItems, "disable-project-items", false, "Whether to disable items in the imported FME Server Projects. If true, items that are new or overwritten will be imported but disabled. If false, project items are imported as defined in the import package.") + cmd.Flags().BoolVar(&f.getSelectable, "get-selectable", false, "Output the selectable items in the import package.") + cmd.Flags().StringVar(&f.selectedItems, "selected-items", "all", "The items to import. Set to \"all\" to import all items, and \"none\" to omit selectable items. Otherwise, this should be a comma separated list of item ids type pairs separated by a colon. e.g. a:b,c:d") + cmd.Flags().BoolVar(&f.interactive, "interactive", false, "Prompt interactively for the selectable items to import (if any exist).") + cmd.Flags().BoolVar(&f.quick, "quick", false, "Import everything in the package by default.") + cmd.Flags().BoolVar(&f.wait, "wait", true, "Wait for import to complete. Set to false to return immediately after the import is started.") + cmd.Flags().StringVar(&f.backupFailureTopic, "failure-topic", "MIGRATION_ASYNC_JOB_FAILURE", "Topic to notify on failure of the backup.") + cmd.Flags().StringVar(&f.backupSuccessTopic, "success-topic", "MIGRATION_ASYNC_JOB_SUCCESS", "Topic to notify on success of the backup.") cmd.Flags().Var(&f.apiVersion, "api-version", "The api version to use when contacting FME Server. Must be one of v3 or v4") + // these flags are only for v3 + cmd.Flags().StringVar(&f.importMode, "import-mode", "", "To import only items in the import package that do not exist on the current instance, specify INSERT. To overwrite items on the current instance with those in the import package, specify UPDATE. Default is INSERT.") + cmd.Flags().StringVar(&f.projectsImportMode, "projects-import-mode", "", "Import mode for projects. To import only projects in the import package that do not exist on the current instance, specify INSERT. To overwrite projects on the current instance with those in the import package, specify UPDATE. If not supplied, importMode will be used.") + cmd.Flags().MarkHidden("api-version") + cmd.Flags().MarkHidden("projects-import-mode") + cmd.Flags().MarkHidden("import-mode") + cmd.MarkFlagsMutuallyExclusive("overwrite", "projects-import-mode") + cmd.MarkFlagsMutuallyExclusive("overwrite", "import-mode") + cmd.MarkFlagsMutuallyExclusive("quick", "get-selectable") + cmd.MarkFlagsMutuallyExclusive("interactive", "get-selectable") + cmd.MarkFlagsMutuallyExclusive("get-selectable", "overwrite") + cmd.MarkFlagsMutuallyExclusive("get-selectable", "pause-notifications") + cmd.MarkFlagsMutuallyExclusive("selected-items", "interactive") + cmd.MarkFlagsMutuallyExclusive("selected-items", "get-selectable") + cmd.MarkFlagsMutuallyExclusive("selected-items", "quick") + cmd.MarkFlagsMutuallyExclusive("quick", "interactive") cmd.RegisterFlagCompletionFunc("api-version", apiVersionFlagCompletion) cmd.MarkFlagRequired("file") @@ -73,17 +193,6 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str return func(cmd *cobra.Command, args []string) error { client := &http.Client{} - // get build to decide if we should use v3 or v4 - // FME Server 2022.0 and later can use v4. Otherwise fall back to v3 - if f.apiVersion == "" { - fmeflowBuild := viper.GetInt("build") - if fmeflowBuild < projectUploadV4BuildThreshold { - f.apiVersion = apiVersionFlagV3 - } else { - f.apiVersion = apiVersionFlagV4 - } - } - url := "" var request http.Request file, err := os.Open(f.file) @@ -93,6 +202,381 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str defer file.Close() if f.apiVersion == "v4" { + // Create a buffer to store our request body as bytes + var requestBody bytes.Buffer + + // Create a multipart writer + multiPartWriter := multipart.NewWriter(&requestBody) + + // Create a form file writer for the file field + fileWriter, err := multiPartWriter.CreateFormFile("file", f.file) + if err != nil { + panic(err) + } + + // Copy the file data to the form file writer + if _, err = io.Copy(fileWriter, file); err != nil { + panic(err) + } + + // Close the multipart writer to get the terminating boundary + if err = multiPartWriter.Close(); err != nil { + panic(err) + } + + url = "/fmeapiv4/migrations/imports/upload" + request, err = buildFmeFlowRequest(url, "POST", &requestBody) + if err != nil { + return err + } + // body as multipart form + request.Header.Set("Content-Type", multiPartWriter.FormDataContentType()) + + if f.quick { + // add the skip preview query parameter for quick import + q := request.URL.Query() + q.Add("skipPreview", strconv.FormatBool(f.quick)) + request.URL.RawQuery = q.Encode() + } + + // execute the upload of the package + response, err := client.Do(&request) + if err != nil { + return err + } else if response.StatusCode != http.StatusCreated { + if response.StatusCode == http.StatusInternalServerError { + return fmt.Errorf("%w: check that the file specified is a valid project file", errors.New(response.Status)) + } else { + return errors.New(response.Status) + } + } + + // read the Location header to get the task id + location := response.Header.Get("Location") + // parse the task id from the location header + // the task id is an integer at the end of the location header + taskId := location[strings.LastIndex(location, "/")+1:] + + var selectedItemsStruct []ProjectSelectedItems + // if this isn't a quick import, we need to get the selectable items by making another rest call. + // also, if it isn't a quick import, it takes a bit of time for the preview to be ready, so we have to wait for it + if !f.quick { + + // We have to do a get on the import to see if the status is ready + ready := false + tries := 0 + url = "/fmeapiv4/migrations/imports/" + taskId + request, err = buildFmeFlowRequest(url, "GET", nil) + if err != nil { + return err + } + + if !jsonOutput && !f.getSelectable { + fmt.Fprint(cmd.OutOrStdout(), "Waiting for preview generation..") + } + + for !ready { + // get the status of the import + response, err = client.Do(&request) + if err != nil { + return err + } else if response.StatusCode != http.StatusOK { + return errors.New(response.Status) + } + + responseData, err := io.ReadAll(response.Body) + if err != nil { + return err + } + + var importStatus ProjectUploadV4 + if err := json.Unmarshal(responseData, &importStatus); err != nil { + return err + } + // check if it is ready + if importStatus.Status == "ready" { + ready = true + } else if importStatus.Status == "generating_preview" { + // if it isn't ready, wait a second and try again + if !jsonOutput && !f.getSelectable { + fmt.Fprint(cmd.OutOrStdout(), ".") + } + time.Sleep(1 * time.Second) + tries++ + } else { + return errors.New("import task did not complete successfully. Status is \"" + importStatus.Status + "\". Please check the FME Flow web interface for the status of the import task") + } + } + // output a newline to cap off the waiting message + if !jsonOutput && !f.getSelectable { + fmt.Fprint(cmd.OutOrStdout(), "\n") + } + + // get the selectable items from the preview + url = "/fmeapiv4/migrations/imports/" + taskId + "/items" + // set up the URL to query + request, err := buildFmeFlowRequest(url, "GET", nil) + if err != nil { + return err + } + + q := request.URL.Query() + q.Add("selectable", "true") + + request.URL.RawQuery = q.Encode() + + response, err = client.Do(&request) + if err != nil { + return err + } else if response.StatusCode != http.StatusOK { + if response.StatusCode == http.StatusInternalServerError { + return fmt.Errorf("%w: check that the file specified is a valid project file", errors.New(response.Status)) + } else { + return errors.New(response.Status) + } + } + + responseData, err := io.ReadAll(response.Body) + if err != nil { + return err + } + + // store the selectable items in a struct + var selectableItems ProjectItems + if err := json.Unmarshal(responseData, &selectableItems); err != nil { + return err + } + + // if we are just outputing the selectable items for this package, just output them, delete the import and return + if f.getSelectable { + // delete the import since we are just getting the selectable items + url = "/fmeapiv4/migrations/imports/" + taskId + request, err = buildFmeFlowRequest(url, "DELETE", nil) + if err != nil { + return err + } + response, err = client.Do(&request) + if err != nil { + return err + } else if response.StatusCode != http.StatusNoContent { + fmt.Fprintln(cmd.OutOrStdout(), "Failed to delete the import task with id "+taskId+". You may need to delete it manually.") + } + + // output the selectable items + if jsonOutput { + prettyJSON, err := prettyPrintJSON(responseData) + if err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), prettyJSON) + } else { + t := table.NewWriter() + t.SetStyle(defaultStyle) + + t.AppendHeader(table.Row{"Id", "Type"}) + + for _, element := range selectableItems.Items { + t.AppendRow(table.Row{element.ID, element.Type}) + } + //if f.noHeaders { + // t.ResetHeaders() + //} + fmt.Fprintln(cmd.OutOrStdout(), t.Render()) + } + return nil + } + + // if we are interactive, we want to prompt the user to select items from the list of selectable ones + if f.interactive { + // store the item ids and types in a string array so that we can prompt the user to select items + var items []string + for _, element := range selectableItems.Items { + items = append(items, element.ID+" ("+element.Type+")") + } + + // prompt the user to select items + // if items is empty, the prompt will automatically be skipped + var selectedItems []string + prompt := &survey.MultiSelect{ + Message: "Select items to import", + Options: items, + } + survey.AskOne(prompt, &selectedItems) + + for _, element := range selectedItems { + // split the string on the space to get the id and type + split := strings.Split(element, " ") + id := split[0] + itemType := strings.Trim(split[1], "()") + selectedItemsStruct = append(selectedItemsStruct, ProjectSelectedItems{ID: id, Type: itemType}) + } + } else { + // if we are not interactive, check the selected items flag to see what to import + if f.selectedItems == "all" { + // loop through all items and format them like we like our input so we can process them in the same way + selectedItemsString := "" + for _, element := range selectableItems.Items { + selectedItemsString += element.ID + ":" + element.Type + "," + } + selectedItemsString = strings.TrimSuffix(selectedItemsString, ",") + f.selectedItems = selectedItemsString + } else if f.selectedItems == "none" { + // if we don't want to select anything, set selectedItemsStruct to an empty list + selectedItemsStruct = []ProjectSelectedItems{} + } + + // parse the selected items and add them to the selectedItemsStruct + if f.selectedItems != "" && f.selectedItems != "none" { + // split the selected items on the comma + split := strings.Split(f.selectedItems, ",") + for _, element := range split { + // split the string on the colon to get the id and type + split := strings.Split(element, ":") + id := split[0] + itemType := split[1] + selectedItemsStruct = append(selectedItemsStruct, ProjectSelectedItems{ID: id, Type: itemType}) + } + } + } + + } + + if !f.getSelectable { + // finally, we can run the import + url = "/fmeapiv4/migrations/imports/" + taskId + "/run" + var run ProjectImportRun + run.Overwrite = f.overwrite + run.PauseNotifications = f.pauseNotifications + run.DisableItems = f.disableProjectItems + run.SelectedItems = selectedItemsStruct + + // if we have selected items, set them here. If we haven't set it, use the default + if selectedItemsStruct != nil { + run.SelectedItems = selectedItemsStruct + } + + // if a topic is specified, add that + if f.backupFailureTopic != "" || f.backupSuccessTopic != "" { + run.Notification = new(ProjectNotification) + run.Notification.Type = "TOPIC" + if f.backupSuccessTopic != "" { + run.Notification.SuccessTopic = f.backupSuccessTopic + } + if f.backupFailureTopic != "" { + run.Notification.FailureTopic = f.backupFailureTopic + } + } else { + run.Notification = nil + } + + // marshal the run struct to json + runJson, err := json.Marshal(run) + if err != nil { + return err + } + + request, err = buildFmeFlowRequest(url, "POST", bytes.NewReader(runJson)) + if err != nil { + return err + } + request.Header.Set("Content-Type", "application/json") + + response, err = client.Do(&request) + if err != nil { + return err + } + if response.StatusCode != http.StatusAccepted { + return errors.New(response.Status) + } else { + if !jsonOutput { + fmt.Fprintln(cmd.OutOrStdout(), "Project Upload task submitted with id: "+taskId) + } else if !f.wait { + // if we are outputting json and not waiting, do a get on the task and output that + url = "/fmeapiv4/migrations/imports/" + taskId + request, err = buildFmeFlowRequest(url, "GET", nil) + if err != nil { + return err + } + + response, err = client.Do(&request) + if err != nil { + return err + } else if response.StatusCode != http.StatusOK { + return errors.New(response.Status) + } + + responseData, err := io.ReadAll(response.Body) + if err != nil { + return err + } + + prettyJSON, err := prettyPrintJSON(responseData) + if err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), prettyJSON) + } + } + + if f.wait { + finished := false + url = "/fmeapiv4/migrations/imports/" + taskId + request, err = buildFmeFlowRequest(url, "GET", nil) + if err != nil { + return err + } + + if !jsonOutput { + fmt.Fprint(cmd.OutOrStdout(), "Waiting for project to finish importing..") + } + var importStatus ProjectUploadV4 + for !finished { + + response, err = client.Do(&request) + if err != nil { + return err + } else if response.StatusCode != http.StatusOK { + return errors.New(response.Status) + } + + responseData, err := io.ReadAll(response.Body) + if err != nil { + return err + } + + if err := json.Unmarshal(responseData, &importStatus); err != nil { + return err + } + //fmt.Fprintln(cmd.OutOrStdout(), importStatus.Status) + if importStatus.Status == "imported" { + finished = true + } else if importStatus.Status != "importing" { + return errors.New("import task did not complete successfully. Please check the FME Flow web interface for the status of the import task") + } else { + if !jsonOutput { + fmt.Fprint(cmd.OutOrStdout(), ".") + } + + time.Sleep(1 * time.Second) + } + } + if !jsonOutput { + fmt.Fprint(cmd.OutOrStdout(), "\n") + fmt.Fprintln(cmd.OutOrStdout(), "Project import complete.") + } else { + jsonData, err := json.Marshal(importStatus) + if err != nil { + return err + } + prettyJSON, err := prettyPrintJSON(jsonData) + if err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), prettyJSON) + } + } + + } } else if f.apiVersion == "v3" { diff --git a/cmd/projects_upload_test.go b/cmd/projects_upload_test.go index 2892888..3b53b41 100644 --- a/cmd/projects_upload_test.go +++ b/cmd/projects_upload_test.go @@ -10,7 +10,7 @@ import ( func TestProjectUpload(t *testing.T) { // standard responses for v3 and v4 - response := `{ + responsev3 := `{ "id": 1 }` projectContents := "Pretend project file" @@ -47,42 +47,92 @@ func TestProjectUpload(t *testing.T) { args: []string{"projects", "upload"}, }, { - name: "upload project", + name: "duplicate flags overwrite projects-import-mode", + wantErrText: "if any flags in the group [overwrite projects-import-mode] are set none of the others can be; [overwrite projects-import-mode] were all set", + args: []string{"projects", "upload", "--file", f.Name(), "--overwrite", "--projects-import-mode", "UPDATE"}, + }, + { + name: "duplicate flags overwrite import-mode", + wantErrText: "if any flags in the group [overwrite import-mode] are set none of the others can be; [import-mode overwrite] were all set", + args: []string{"projects", "upload", "--file", f.Name(), "--overwrite", "--import-mode", "UPDATE"}, + }, + { + name: "duplicate flags quick get-selectable", + wantErrText: "if any flags in the group [quick get-selectable] are set none of the others can be; [get-selectable quick] were all set", + args: []string{"projects", "upload", "--file", f.Name(), "--quick", "--get-selectable"}, + }, + { + name: "duplicate flags interactive get-selectable", + wantErrText: "if any flags in the group [interactive get-selectable] are set none of the others can be; [get-selectable interactive] were all set", + args: []string{"projects", "upload", "--file", f.Name(), "--interactive", "--get-selectable"}, + }, + { + name: "duplicate flags overwrite get-selectable", + wantErrText: "if any flags in the group [get-selectable overwrite] are set none of the others can be; [get-selectable overwrite] were all set", + args: []string{"projects", "upload", "--file", f.Name(), "--overwrite", "--get-selectable"}, + }, + { + name: "duplicate flags pause-notifications get-selectable", + wantErrText: "if any flags in the group [get-selectable pause-notifications] are set none of the others can be; [get-selectable pause-notifications] were all set", + args: []string{"projects", "upload", "--file", f.Name(), "--pause-notifications", "--get-selectable"}, + }, + { + name: "duplicate flags selected-items interactive", + wantErrText: "if any flags in the group [selected-items interactive] are set none of the others can be; [interactive selected-items] were all set", + args: []string{"projects", "upload", "--file", f.Name(), "--selected-items=[]", "--interactive"}, + }, + { + name: "duplicate flags selected-items get-selectable", + wantErrText: "if any flags in the group [selected-items get-selectable] are set none of the others can be; [get-selectable selected-items] were all set", + args: []string{"projects", "upload", "--file", f.Name(), "--selected-items=[]", "--get-selectable"}, + }, + { + name: "duplicate flags selected-items quick", + wantErrText: "if any flags in the group [selected-items quick] are set none of the others can be; [quick selected-items] were all set", + args: []string{"projects", "upload", "--file", f.Name(), "--selected-items=[]", "--quick"}, + }, + { + name: "duplicate flags interactive quick", + wantErrText: "if any flags in the group [quick interactive] are set none of the others can be; [interactive quick] were all set", + args: []string{"projects", "upload", "--file", f.Name(), "--interactive", "--quick"}, + }, + { + name: "upload project V3", statusCode: http.StatusOK, - args: []string{"projects", "upload", "--file", f.Name()}, - body: response, + args: []string{"projects", "upload", "--file", f.Name(), "--api-version", "v3"}, + body: responsev3, wantOutputRegex: "Project Upload task submitted with id: 1", }, { - name: "import mode", + name: "import mode V3", statusCode: http.StatusOK, - args: []string{"projects", "upload", "--file", f.Name(), "--import-mode", "UPDATE"}, - body: response, + args: []string{"projects", "upload", "--file", f.Name(), "--import-mode", "UPDATE", "--api-version", "v3"}, + body: responsev3, wantOutputRegex: "Project Upload task submitted with id: 1", wantFormParams: map[string]string{"importMode": "UPDATE"}, }, { - name: "projects import mode", + name: "projects import mode V3", statusCode: http.StatusOK, - args: []string{"projects", "upload", "--file", f.Name(), "--projects-import-mode", "UPDATE"}, - body: response, + args: []string{"projects", "upload", "--file", f.Name(), "--projects-import-mode", "UPDATE", "--api-version", "v3"}, + body: responsev3, wantOutputRegex: "Project Upload task submitted with id: 1", wantFormParams: map[string]string{"projectsImportMode": "UPDATE"}, wantBodyRegEx: projectContents, }, { - name: "pause-notifications", + name: "pause-notifications V3", statusCode: http.StatusOK, - args: []string{"projects", "upload", "--file", f.Name(), "--pause-notifications"}, - body: response, + args: []string{"projects", "upload", "--file", f.Name(), "--pause-notifications", "--api-version", "v3"}, + body: responsev3, wantOutputRegex: "Project Upload task submitted with id: 1", wantFormParams: map[string]string{"pauseNotifications": "true"}, }, { - name: "disable project items", + name: "disable project items V3", statusCode: http.StatusOK, - args: []string{"projects", "upload", "--file", f.Name(), "--disable-project-items"}, - body: response, + args: []string{"projects", "upload", "--file", f.Name(), "--disable-project-items", "--api-version", "v3"}, + body: responsev3, wantOutputRegex: "Project Upload task submitted with id: 1", wantFormParams: map[string]string{"disableProjectItems": "true"}, }, From f2d3f713ff7ec2eaddcadda3fbd65696bc1beb8c Mon Sep 17 00:00:00 2001 From: Grant Arnold Date: Tue, 12 Mar 2024 08:15:46 -0700 Subject: [PATCH 3/5] Finish implementation of project upload using v4. --- cmd/projects_upload.go | 61 +++++-- cmd/projects_upload_test.go | 322 +++++++++++++++++++++++++++++++++++- cmd/testFunctions.go | 2 +- 3 files changed, 369 insertions(+), 16 deletions(-) diff --git a/cmd/projects_upload.go b/cmd/projects_upload.go index 9f06c2c..2a4415c 100644 --- a/cmd/projects_upload.go +++ b/cmd/projects_upload.go @@ -9,6 +9,7 @@ import ( "mime/multipart" "net/http" "os" + "regexp" "strconv" "strings" "time" @@ -84,14 +85,14 @@ type ProjectSelectedItems struct { } type ProjectUploadV4 struct { - JobID int `json:"jobId"` - Status string `json:"status"` - Owner string `json:"owner"` - OwnerID string `json:"ownerID"` - Requested time.Time `json:"requested"` - Generated time.Time `json:"generated"` - FileName string `json:"fileName"` - Request interface{} `json:"request"` + JobID int `json:"jobId"` + Status string `json:"status"` + Owner string `json:"owner"` + OwnerID string `json:"ownerID"` + Requested time.Time `json:"requested"` + Generated time.Time `json:"generated"` + FileName string `json:"fileName"` + Request ProjectImportRun `json:"request"` } var projectUploadV4BuildThreshold = 23766 @@ -101,7 +102,11 @@ func newProjectUploadCmd() *cobra.Command { cmd := &cobra.Command{ Use: "upload", Short: "Imports FME Server Projects from a downloaded package.", - Long: "Imports FME Server Projects from a downloaded package. Useful for moving a project from one FME Server to another.", + Long: `Imports FME Server Projects from a downloaded package. The upload happens in two steps. The package is uploaded to the server, a preview is generated that contains the list of items, and then the import is run. This command can be run using a few different modes. +- Using the --get-selectable flag will just generate the preview and output the selectable items in the package and then delete the import +- Using the --quick flag will skip the preview and import everything in the package by default. +- Using the --interactive flag will prompt the user to select items to import from the list of selectable items if any exist +- Using the --selected-items flag will import only the items specified. The default is to import all items in the package.`, PreRunE: func(cmd *cobra.Command, args []string) error { // get build to decide if we should use v3 or v4 // FME Server 2022.0 and later can use v4. Otherwise fall back to v3 @@ -129,7 +134,7 @@ func newProjectUploadCmd() *cobra.Command { return errors.New("cannot set the importMode flag when using the V4 API") } if f.projectsImportMode != "" { - return errors.New("cannot set the projectsImportMode flag when using the V3 API") + return errors.New("cannot set the projectsImportMode flag when using the V4 API") } } @@ -146,11 +151,23 @@ func newProjectUploadCmd() *cobra.Command { return nil }, Example: ` - # Restore from a backup in a local file + # Upload a project and import all selectable items if any exist fmeflow projects upload --file ProjectPackage.fsproject - # Restore from a backup in a local file without overwriting - fmeflow projects upload --file ProjectPackage.fsproject --overwrite=false`, + # Upload a project without overwriting existing items + fmeflow projects upload --file ProjectPackage.fsproject --overwrite=false + + # Upload a project and perform a quick import + fmeflow projects upload --file ProjectPackage.fsproject --quick + + # Upload a project and be prompted for which items to import of the selectable items + fmeflow projects upload --file ProjectPackage.fsproject --interactive + + # Upload a project and get the list of selectable items + fmeflow projects upload --file ProjectPackage.fsproject --get-selectable + + # Upload a project and import only the specified selectable items + fme projects upload --file ProjectPackage.fsproject --selected-items="mysqldb:connection,slack con:connector"`, Args: NoArgs, RunE: projectUploadRun(&f), } @@ -427,6 +444,12 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str // parse the selected items and add them to the selectedItemsStruct if f.selectedItems != "" && f.selectedItems != "none" { + // validate that the selected items are the correct format + match, _ := regexp.MatchString(`^([^:,]+:[^:,]+,)*([^:,]+:[^:,]+)$`, f.selectedItems) + if !match { + return errors.New("invalid selected items. Must be a comma separated list of item ids and types. e.g. item1:itemtype1,item2:itemtype2") + } + // split the selected items on the comma split := strings.Split(f.selectedItems, ",") for _, element := range split { @@ -434,6 +457,18 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str split := strings.Split(element, ":") id := split[0] itemType := split[1] + // verify that the selected items are in the list of selectable items + found := false + for _, selectableItem := range selectableItems.Items { + if selectableItem.ID == id && selectableItem.Type == itemType { + found = true + break + } + } + if !found { + return errors.New("selected item " + id + " (" + itemType + ") is not in the list of selectable items") + } + selectedItemsStruct = append(selectedItemsStruct, ProjectSelectedItems{ID: id, Type: itemType}) } } diff --git a/cmd/projects_upload_test.go b/cmd/projects_upload_test.go index 3b53b41..5eba804 100644 --- a/cmd/projects_upload_test.go +++ b/cmd/projects_upload_test.go @@ -1,9 +1,14 @@ package cmd import ( + "encoding/json" + "io" "net/http" + "net/http/httptest" "os" + "strings" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -15,6 +20,68 @@ func TestProjectUpload(t *testing.T) { }` projectContents := "Pretend project file" + ProjectNotificationStruct := ProjectNotification{ + Type: "TOPIC", + SuccessTopic: "MIGRATION_ASYNC_JOB_SUCCESS", + FailureTopic: "MIGRATION_ASYNC_JOB_FAILURE", + } + + ProjectGetStruct := ProjectUploadV4{ + JobID: 1, + Status: "importing", + Owner: "admin", + OwnerID: "fb2dd313-e5cf-432e-a24a-814e46929ab7", + Requested: time.Date(2024, 3, 8, 19, 53, 25, 518000000, time.UTC), + Generated: time.Date(2024, 3, 8, 19, 53, 25, 518000000, time.UTC), + FileName: "test", + Request: ProjectImportRun{ + FallbackOwnerID: "", + Overwrite: true, + PauseNotifications: true, + DisableItems: false, + Notification: &ProjectNotificationStruct, + SelectedItems: nil, + }, + } + + ProjectItemsJson := `{ + "items": [ + { + "id": "test", + "jobId": 68, + "name": "test", + "type": "deploymentParameter", + "ownerId": "fb2dd313-e5cf-432e-a24a-814e46929ab7", + "ownerName": "admin", + "ownerStatus": "id_match", + "originalOwner": "admin", + "selected": true, + "existing": true, + "previewAction": "skipped", + "action": "unknown", + "source": "project" + }, + { + "id": "6a3ebaf9-e537-4aff-9be0-b8d88542069e", + "jobId": 68, + "name": "author", + "type": "user", + "ownerId": null, + "ownerName": null, + "ownerStatus": "none", + "originalOwner": null, + "selected": true, + "existing": true, + "previewAction": "overwritten", + "action": "unknown", + "source": "project" + } + ], + "totalCount": 2, + "limit": 100, + "offset": 0 + }` + // generate random file to restore from f, err := os.CreateTemp("", "fmeflow-project") require.NoError(t, err) @@ -22,6 +89,184 @@ func TestProjectUpload(t *testing.T) { err = os.WriteFile(f.Name(), []byte(projectContents), 0644) require.NoError(t, err) + // state variable for our custom http handler + getCount := 0 + + // this is the generic mock up for this test which will respond similar to how FME Flow should respond + customHttpServerHandler := func(w http.ResponseWriter, r *http.Request) { + + if strings.Contains(r.URL.Path, "upload") { + // set a location header + w.Header().Set("Location", "http://localhost:8080/fmeapiv4/migrations/imports/1") + w.WriteHeader(http.StatusCreated) + + // check if there is a URL argument + r.ParseForm() + urlParams := r.Form + if urlParams.Get("skipPreview") != "" { + // set status to generating preview + ProjectGetStruct.Status = "generating_preview" + } else { + // set status to ready + ProjectGetStruct.Status = "ready" + } + + } else if strings.Contains(r.URL.Path, "run") && r.Method == "POST" { + w.WriteHeader(http.StatusAccepted) + require.Contains(t, r.URL.Path, "migrations/imports/1/run") + // set status to importing + ProjectGetStruct.Status = "importing" + } else if strings.Contains(r.URL.Path, "migrations/imports/1/items") && r.Method == "GET" { + w.WriteHeader(http.StatusOK) + // return the test list of selectable items + _, err := w.Write([]byte(ProjectItemsJson)) + require.NoError(t, err) + // set status to importing + ProjectGetStruct.Status = "importing" + } else if strings.Contains(r.URL.Path, "migrations/imports/1") && r.Method == "GET" { + w.WriteHeader(http.StatusOK) + if getCount < 1 { + getCount++ + } else { + if ProjectGetStruct.Status == "importing" { + // set status to imported + ProjectGetStruct.Status = "imported" + } else if ProjectGetStruct.Status == "generating_preview" { + //set status to ready + ProjectGetStruct.Status = "ready" + } + getCount = 0 + } + // marshal the struct to json + projectGetJson, err := json.Marshal(ProjectGetStruct) + require.NoError(t, err) + + // write the json to the response + _, err = w.Write(projectGetJson) + require.NoError(t, err) + } else if strings.Contains(r.URL.Path, "migrations/imports/1") && r.Method == "DELETE" { + w.WriteHeader(http.StatusNoContent) + } else { + w.WriteHeader(http.StatusNotFound) + } + } + + // this will check for the correct settings for the quick test, then call the custom handler + quickHttpServerHandler := func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "upload") { + // check if the quick flag is set on upload + r.ParseForm() + urlParams := r.Form + require.True(t, urlParams.Has("skipPreview")) + } + customHttpServerHandler(w, r) + } + + // this will check for the correct settings for the overwrite test, then call the custom handler + overwriteHttpServerHandler := func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "run") && r.Method == "POST" { + // check if the overwite is in the json body + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + var result map[string]interface{} + err = json.Unmarshal(body, &result) + require.NoError(t, err) + require.Equal(t, true, result["overwrite"]) + } + customHttpServerHandler(w, r) + } + + // this will check for the correct settings for the overwrite test, then call the custom handler + pauseNotificationsHttpServerHandler := func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "run") && r.Method == "POST" { + // check if the overwite is in the json body + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + var result map[string]interface{} + err = json.Unmarshal(body, &result) + require.NoError(t, err) + require.Equal(t, true, result["pauseNotifications"]) + } + customHttpServerHandler(w, r) + } + + // this will check for the correct settings for the disableItems test, then call the custom handler + disableItemsHttpServerHandler := func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "run") && r.Method == "POST" { + // check if the overwite is in the json body + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + var result map[string]interface{} + err = json.Unmarshal(body, &result) + require.NoError(t, err) + require.Equal(t, true, result["disableItems"]) + } + customHttpServerHandler(w, r) + } + + // this will check for the correct settings for the topics test, then call the custom handler + topicsHttpServerHandler := func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "run") && r.Method == "POST" { + // check if the overwite is in the json body + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + var result ProjectImportRun + err = json.Unmarshal(body, &result) + require.NoError(t, err) + require.Equal(t, "SUCCESS_TOPIC", result.Notification.SuccessTopic) + require.Equal(t, "FAILURE_TOPIC", result.Notification.FailureTopic) + } + customHttpServerHandler(w, r) + } + + // this will check that the selected items are set correctly + selectedItemsAllHttpServerHandler := func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "run") && r.Method == "POST" { + // check if the overwite is in the json body + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + var result ProjectImportRun + err = json.Unmarshal(body, &result) + require.NoError(t, err) + require.Equal(t, 2, len(result.SelectedItems)) + require.Equal(t, "test", result.SelectedItems[0].ID) + require.Equal(t, "deploymentParameter", result.SelectedItems[0].Type) + require.Equal(t, "6a3ebaf9-e537-4aff-9be0-b8d88542069e", result.SelectedItems[1].ID) + require.Equal(t, "user", result.SelectedItems[1].Type) + } + customHttpServerHandler(w, r) + } + + // this will check that only the test item is set + selectedItemsListHttpServerHandler := func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "run") && r.Method == "POST" { + // check if the overwite is in the json body + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + var result ProjectImportRun + err = json.Unmarshal(body, &result) + require.NoError(t, err) + require.Equal(t, 1, len(result.SelectedItems)) + require.Equal(t, "test", result.SelectedItems[0].ID) + require.Equal(t, "deploymentParameter", result.SelectedItems[0].Type) + } + customHttpServerHandler(w, r) + } + + // check that no selected items are set + selectedItemsNoneHttpServerHandler := func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "run") && r.Method == "POST" { + // check if the overwite is in the json body + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + var result ProjectImportRun + err = json.Unmarshal(body, &result) + require.NoError(t, err) + require.Equal(t, 0, len(result.SelectedItems)) + } + customHttpServerHandler(w, r) + } + cases := []testCase{ { name: "unknown flag", @@ -49,12 +294,12 @@ func TestProjectUpload(t *testing.T) { { name: "duplicate flags overwrite projects-import-mode", wantErrText: "if any flags in the group [overwrite projects-import-mode] are set none of the others can be; [overwrite projects-import-mode] were all set", - args: []string{"projects", "upload", "--file", f.Name(), "--overwrite", "--projects-import-mode", "UPDATE"}, + args: []string{"projects", "upload", "--file", f.Name(), "--overwrite", "--projects-import-mode", "UPDATE", "--api-version", "v3"}, }, { name: "duplicate flags overwrite import-mode", wantErrText: "if any flags in the group [overwrite import-mode] are set none of the others can be; [import-mode overwrite] were all set", - args: []string{"projects", "upload", "--file", f.Name(), "--overwrite", "--import-mode", "UPDATE"}, + args: []string{"projects", "upload", "--file", f.Name(), "--overwrite", "--import-mode", "UPDATE", "--api-version", "v3"}, }, { name: "duplicate flags quick get-selectable", @@ -96,6 +341,79 @@ func TestProjectUpload(t *testing.T) { wantErrText: "if any flags in the group [quick interactive] are set none of the others can be; [interactive quick] were all set", args: []string{"projects", "upload", "--file", f.Name(), "--interactive", "--quick"}, }, + { + name: "upload project V4 quick", + args: []string{"projects", "upload", "--file", f.Name(), "--quick"}, + httpServer: httptest.NewServer(http.HandlerFunc(quickHttpServerHandler)), + wantOutputRegex: "Project import complete.", + }, + { + name: "upload project V4 overwrite", + args: []string{"projects", "upload", "--file", f.Name(), "--overwrite"}, + httpServer: httptest.NewServer(http.HandlerFunc(overwriteHttpServerHandler)), + wantOutputRegex: "Project import complete.", + }, + { + name: "upload project V4 pause-notifications", + args: []string{"projects", "upload", "--file", f.Name(), "--pause-notifications"}, + httpServer: httptest.NewServer(http.HandlerFunc(pauseNotificationsHttpServerHandler)), + wantOutputRegex: "Project import complete.", + }, + { + name: "upload project V4 disable-project-items", + args: []string{"projects", "upload", "--file", f.Name(), "--disable-project-items"}, + httpServer: httptest.NewServer(http.HandlerFunc(disableItemsHttpServerHandler)), + wantOutputRegex: "Project import complete.", + }, + { + name: "upload project V4 get-selectable", + args: []string{"projects", "upload", "--file", f.Name(), "--get-selectable"}, + httpServer: httptest.NewServer(http.HandlerFunc(customHttpServerHandler)), + wantOutputRegex: "^[\\s]*ID[\\s]*TYPE[\\s]*test[\\s]*deploymentParameter[\\s]*6a3ebaf9-e537-4aff-9be0-b8d88542069e[\\s]*user[\\s]*$", + }, + { + name: "upload project V4 get-selectable json", + args: []string{"projects", "upload", "--file", f.Name(), "--get-selectable", "--json"}, + httpServer: httptest.NewServer(http.HandlerFunc(customHttpServerHandler)), + wantOutputJson: ProjectItemsJson, + }, + { + name: "upload project V4 set success and failure topics", + args: []string{"projects", "upload", "--file", f.Name(), "--success-topic", "SUCCESS_TOPIC", "--failure-topic", "FAILURE_TOPIC"}, + httpServer: httptest.NewServer(http.HandlerFunc(topicsHttpServerHandler)), + }, + { + name: "upload project V4 selected items all", + args: []string{"projects", "upload", "--file", f.Name(), "--selected-items", "all"}, + httpServer: httptest.NewServer(http.HandlerFunc(selectedItemsAllHttpServerHandler)), + }, + { + name: "upload project V4 selected items none", + args: []string{"projects", "upload", "--file", f.Name(), "--selected-items", "none"}, + httpServer: httptest.NewServer(http.HandlerFunc(selectedItemsNoneHttpServerHandler)), + }, + { + name: "upload project V4 selected items list", + args: []string{"projects", "upload", "--file", f.Name(), "--selected-items", "test:deploymentParameter"}, + httpServer: httptest.NewServer(http.HandlerFunc(selectedItemsListHttpServerHandler)), + }, + { + name: "upload project V4 selected items list 2", + args: []string{"projects", "upload", "--file", f.Name(), "--selected-items", "test:deploymentParameter,6a3ebaf9-e537-4aff-9be0-b8d88542069e:user"}, + httpServer: httptest.NewServer(http.HandlerFunc(selectedItemsAllHttpServerHandler)), + }, + { + name: "upload project V4 invalid selected items syntax", + args: []string{"projects", "upload", "--file", f.Name(), "--selected-items", "test:deploymentParameter,6a3ebaf9-e537-4aff-9be0-b8d88542069e"}, + wantErrText: "invalid selected items. Must be a comma separated list of item ids and types. e.g. item1:itemtype1,item2:itemtype2", + httpServer: httptest.NewServer(http.HandlerFunc(customHttpServerHandler)), + }, + { + name: "upload project V4 invalid selected items in package", + args: []string{"projects", "upload", "--file", f.Name(), "--selected-items", "test:deploymentParameter,author:user"}, + wantErrText: "selected item author (user) is not in the list of selectable items", + httpServer: httptest.NewServer(http.HandlerFunc(customHttpServerHandler)), + }, { name: "upload project V3", statusCode: http.StatusOK, diff --git a/cmd/testFunctions.go b/cmd/testFunctions.go index 9f3467d..79ed4e0 100644 --- a/cmd/testFunctions.go +++ b/cmd/testFunctions.go @@ -82,7 +82,7 @@ func runTests(tcs []testCase, t *testing.T) { if tc.fmeflowBuild != 0 { viper.Set("build", tc.fmeflowBuild) } else { - viper.Set("build", 23159) + viper.Set("build", 23776) } } From c2aa3ce6da8e5a92b04e5ba4e309687c5c99e3c2 Mon Sep 17 00:00:00 2001 From: Grant Arnold Date: Tue, 12 Mar 2024 09:40:07 -0700 Subject: [PATCH 4/5] Some cleanup. --- cmd/projects_upload.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/cmd/projects_upload.go b/cmd/projects_upload.go index 2a4415c..cd3d764 100644 --- a/cmd/projects_upload.go +++ b/cmd/projects_upload.go @@ -206,6 +206,7 @@ func newProjectUploadCmd() *cobra.Command { return cmd } + func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []string) error { return func(cmd *cobra.Command, args []string) error { client := &http.Client{} @@ -228,17 +229,17 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str // Create a form file writer for the file field fileWriter, err := multiPartWriter.CreateFormFile("file", f.file) if err != nil { - panic(err) + return err } // Copy the file data to the form file writer if _, err = io.Copy(fileWriter, file); err != nil { - panic(err) + return err } // Close the multipart writer to get the terminating boundary if err = multiPartWriter.Close(); err != nil { - panic(err) + return err } url = "/fmeapiv4/migrations/imports/upload" @@ -292,6 +293,7 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str fmt.Fprint(cmd.OutOrStdout(), "Waiting for preview generation..") } + // we have to loop until the preview is done generating for !ready { // get the status of the import response, err = client.Do(&request) @@ -314,7 +316,7 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str if importStatus.Status == "ready" { ready = true } else if importStatus.Status == "generating_preview" { - // if it isn't ready, wait a second and try again + // if it is still generating the preview, wait a second and try again if !jsonOutput && !f.getSelectable { fmt.Fprint(cmd.OutOrStdout(), ".") } @@ -324,6 +326,7 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str return errors.New("import task did not complete successfully. Status is \"" + importStatus.Status + "\". Please check the FME Flow web interface for the status of the import task") } } + // output a newline to cap off the waiting message if !jsonOutput && !f.getSelectable { fmt.Fprint(cmd.OutOrStdout(), "\n") @@ -480,10 +483,10 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str // finally, we can run the import url = "/fmeapiv4/migrations/imports/" + taskId + "/run" var run ProjectImportRun + // set the run struct run.Overwrite = f.overwrite run.PauseNotifications = f.pauseNotifications run.DisableItems = f.disableProjectItems - run.SelectedItems = selectedItemsStruct // if we have selected items, set them here. If we haven't set it, use the default if selectedItemsStruct != nil { @@ -553,6 +556,7 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str } } + // if we are waiting for the import to complete, we have to loop until it is done if f.wait { finished := false url = "/fmeapiv4/migrations/imports/" + taskId @@ -582,7 +586,7 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str if err := json.Unmarshal(responseData, &importStatus); err != nil { return err } - //fmt.Fprintln(cmd.OutOrStdout(), importStatus.Status) + if importStatus.Status == "imported" { finished = true } else if importStatus.Status != "importing" { From 7df28dbc39de1e1577f16e003f30b30a71939d4b Mon Sep 17 00:00:00 2001 From: Grant Arnold Date: Wed, 13 Mar 2024 22:58:34 +0000 Subject: [PATCH 5/5] Code review changes. --- cmd/projects_upload.go | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/cmd/projects_upload.go b/cmd/projects_upload.go index cd3d764..8aac095 100644 --- a/cmd/projects_upload.go +++ b/cmd/projects_upload.go @@ -101,8 +101,8 @@ func newProjectUploadCmd() *cobra.Command { f := projectUploadFlags{} cmd := &cobra.Command{ Use: "upload", - Short: "Imports FME Server Projects from a downloaded package.", - Long: `Imports FME Server Projects from a downloaded package. The upload happens in two steps. The package is uploaded to the server, a preview is generated that contains the list of items, and then the import is run. This command can be run using a few different modes. + Short: "Imports FME Flow Projects from a downloaded package.", + Long: `Imports FME Flow Projects from a downloaded package. The upload happens in two steps. The package is uploaded to the server, a preview is generated that contains the list of items, and then the import is run. This command can be run using a few different modes. - Using the --get-selectable flag will just generate the preview and output the selectable items in the package and then delete the import - Using the --quick flag will skip the preview and import everything in the package by default. - Using the --interactive flag will prompt the user to select items to import from the list of selectable items if any exist @@ -275,7 +275,7 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str // the task id is an integer at the end of the location header taskId := location[strings.LastIndex(location, "/")+1:] - var selectedItemsStruct []ProjectSelectedItems + var selectedItemsSlice []ProjectSelectedItems // if this isn't a quick import, we need to get the selectable items by making another rest call. // also, if it isn't a quick import, it takes a bit of time for the preview to be ready, so we have to wait for it if !f.quick { @@ -349,11 +349,7 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str if err != nil { return err } else if response.StatusCode != http.StatusOK { - if response.StatusCode == http.StatusInternalServerError { - return fmt.Errorf("%w: check that the file specified is a valid project file", errors.New(response.Status)) - } else { - return errors.New(response.Status) - } + return errors.New("error retrieving items: " + response.Status) } responseData, err := io.ReadAll(response.Body) @@ -398,9 +394,7 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str for _, element := range selectableItems.Items { t.AppendRow(table.Row{element.ID, element.Type}) } - //if f.noHeaders { - // t.ResetHeaders() - //} + fmt.Fprintln(cmd.OutOrStdout(), t.Render()) } return nil @@ -428,7 +422,7 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str split := strings.Split(element, " ") id := split[0] itemType := strings.Trim(split[1], "()") - selectedItemsStruct = append(selectedItemsStruct, ProjectSelectedItems{ID: id, Type: itemType}) + selectedItemsSlice = append(selectedItemsSlice, ProjectSelectedItems{ID: id, Type: itemType}) } } else { // if we are not interactive, check the selected items flag to see what to import @@ -441,11 +435,11 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str selectedItemsString = strings.TrimSuffix(selectedItemsString, ",") f.selectedItems = selectedItemsString } else if f.selectedItems == "none" { - // if we don't want to select anything, set selectedItemsStruct to an empty list - selectedItemsStruct = []ProjectSelectedItems{} + // if we don't want to select anything, set selectedItemsSlice to an empty list + selectedItemsSlice = []ProjectSelectedItems{} } - // parse the selected items and add them to the selectedItemsStruct + // parse the selected items and add them to the selectedItemsSlice if f.selectedItems != "" && f.selectedItems != "none" { // validate that the selected items are the correct format match, _ := regexp.MatchString(`^([^:,]+:[^:,]+,)*([^:,]+:[^:,]+)$`, f.selectedItems) @@ -472,7 +466,7 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str return errors.New("selected item " + id + " (" + itemType + ") is not in the list of selectable items") } - selectedItemsStruct = append(selectedItemsStruct, ProjectSelectedItems{ID: id, Type: itemType}) + selectedItemsSlice = append(selectedItemsSlice, ProjectSelectedItems{ID: id, Type: itemType}) } } } @@ -489,8 +483,8 @@ func projectUploadRun(f *projectUploadFlags) func(cmd *cobra.Command, args []str run.DisableItems = f.disableProjectItems // if we have selected items, set them here. If we haven't set it, use the default - if selectedItemsStruct != nil { - run.SelectedItems = selectedItemsStruct + if selectedItemsSlice != nil { + run.SelectedItems = selectedItemsSlice } // if a topic is specified, add that