-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fetch: Asset download utility (#1256)
* fetch: Initial skeleton * fetch: Define config scheme * fetch: Load config file * fetch: Build target-path * fetch: Gather asset count * fetch: Create rest session * cmake: Add run.fetch target * net: Fix bug in net_rest_destroy * fetch: Initial json config * fetch: Add origin license * fetch: Initial fetch logic * fetch: Refactor fetch logic * fetch: Log when asset-save fails * fetch: Log duration * fetch: Fix target path create logic * assets: Configure fetch * cmake: Remove asset downloading * fetch: Fix failing to save asset * fetch: Add verbose log option * fetch: Fix fetch ret-code * fetch: Support origin auth * fetch: Reduce sleep time * fetch: Minor cleanup
- Loading branch information
1 parent
6463cd9
commit 9f9a63b
Showing
9 changed files
with
559 additions
and
301 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,279 @@ | ||
| #include "app_cli.h" | ||
| #include "cli_app.h" | ||
| #include "cli_help.h" | ||
| #include "cli_parse.h" | ||
| #include "cli_read.h" | ||
| #include "cli_validate.h" | ||
| #include "core_alloc.h" | ||
| #include "core_array.h" | ||
| #include "core_dynstring.h" | ||
| #include "core_file.h" | ||
| #include "core_math.h" | ||
| #include "core_path.h" | ||
| #include "core_thread.h" | ||
| #include "core_time.h" | ||
| #include "data_read.h" | ||
| #include "data_utils.h" | ||
| #include "log_logger.h" | ||
| #include "log_sink_json.h" | ||
| #include "log_sink_pretty.h" | ||
| #include "net_http.h" | ||
| #include "net_init.h" | ||
| #include "net_rest.h" | ||
| #include "net_result.h" | ||
|
|
||
| /** | ||
| * Fetch - Utility to download external assets. | ||
| */ | ||
|
|
||
| #define fetch_worker_count 2 | ||
|
|
||
| typedef struct { | ||
| String host; | ||
| String license; | ||
| String rootUri; | ||
| String authUser, authPass; | ||
| HeapArray_t(String) assets; | ||
| } FetchOrigin; | ||
|
|
||
| typedef struct { | ||
| String targetPath; | ||
| HeapArray_t(FetchOrigin) origins; | ||
| } FetchConfig; | ||
|
|
||
| static DataMeta g_fetchConfigMeta; | ||
|
|
||
| static void fetch_data_init(void) { | ||
| // clang-format off | ||
| data_reg_struct_t(g_dataReg, FetchOrigin); | ||
| data_reg_field_t(g_dataReg, FetchOrigin, host, data_prim_t(String), .flags = DataFlags_NotEmpty); | ||
| data_reg_field_t(g_dataReg, FetchOrigin, license, data_prim_t(String), .flags = DataFlags_Opt); | ||
| data_reg_field_t(g_dataReg, FetchOrigin, rootUri, data_prim_t(String)); | ||
| data_reg_field_t(g_dataReg, FetchOrigin, authUser, data_prim_t(String), .flags = DataFlags_Opt | DataFlags_NotEmpty); | ||
| data_reg_field_t(g_dataReg, FetchOrigin, authPass, data_prim_t(String), .flags = DataFlags_Opt | DataFlags_NotEmpty); | ||
| data_reg_field_t(g_dataReg, FetchOrigin, assets, data_prim_t(String), .container = DataContainer_HeapArray, .flags = DataFlags_NotEmpty); | ||
|
|
||
| data_reg_struct_t(g_dataReg, FetchConfig); | ||
| data_reg_field_t(g_dataReg, FetchConfig, targetPath, data_prim_t(String)); | ||
| data_reg_field_t(g_dataReg, FetchConfig, origins, t_FetchOrigin, .container = DataContainer_HeapArray); | ||
| // clang-format on | ||
|
|
||
| g_fetchConfigMeta = data_meta_t(t_FetchConfig); | ||
| } | ||
|
|
||
| static bool fetch_config_load(const String path, FetchConfig* out) { | ||
| // Open the file handle. | ||
| bool success = false; | ||
| File* file = null; | ||
| FileResult fileRes; | ||
| if ((fileRes = file_create(g_allocScratch, path, FileMode_Open, FileAccess_Read, &file))) { | ||
| log_e("Failed to open config file", log_param("err", fmt_text(file_result_str(fileRes)))); | ||
| goto Ret; | ||
| } | ||
|
|
||
| // Map the file data. | ||
| String fileData; | ||
| if (UNLIKELY(fileRes = file_map(file, &fileData, FileHints_Prefetch))) { | ||
| log_e("Failed to map config file", log_param("err", fmt_text(file_result_str(fileRes)))); | ||
| goto Ret; | ||
| } | ||
|
|
||
| // Parse the json. | ||
| DataReadResult result; | ||
| const Mem outMem = mem_create(out, sizeof(FetchConfig)); | ||
| data_read_json(g_dataReg, fileData, g_allocHeap, g_fetchConfigMeta, outMem, &result); | ||
| if (UNLIKELY(result.error)) { | ||
| log_e("Failed to parse config file", log_param("err", fmt_text(result.errorMsg))); | ||
| goto Ret; | ||
| } | ||
| success = true; | ||
|
|
||
| Ret: | ||
| if (file) { | ||
| file_destroy(file); | ||
| } | ||
| return success; | ||
| } | ||
|
|
||
| static void fetch_config_destroy(FetchConfig* cfg) { | ||
| data_destroy(g_dataReg, g_allocHeap, g_fetchConfigMeta, mem_create(cfg, sizeof(FetchConfig))); | ||
| } | ||
|
|
||
| static u32 fetch_config_max_origin_assets(FetchConfig* cfg) { | ||
| u32 res = 0; | ||
| heap_array_for_t(cfg->origins, FetchOrigin, origin) { | ||
| res = math_max(res, (u32)origin->assets.count); | ||
| } | ||
| return res; | ||
| } | ||
|
|
||
| static String fetch_config_uri_scratch(const FetchOrigin* origin, const String asset) { | ||
| DynString result = dynstring_create(g_allocScratch, 256); | ||
| if (!string_starts_with(origin->rootUri, string_lit("/"))) { | ||
| dynstring_append_char(&result, '/'); | ||
| } | ||
| dynstring_append(&result, origin->rootUri); | ||
| if (!string_ends_with(dynstring_view(&result), string_lit("/"))) { | ||
| dynstring_append_char(&result, '/'); | ||
| } | ||
| if (string_starts_with(asset, string_lit("/"))) { | ||
| dynstring_append(&result, string_consume(asset, 1)); | ||
| } else { | ||
| dynstring_append(&result, asset); | ||
| } | ||
| return dynstring_view(&result); | ||
| } | ||
|
|
||
| static NetHttpFlags fetch_http_flags(void) { | ||
| /** | ||
| * Enable Tls transport but do not enable certificate validation. | ||
| * This means traffic is encrypted and people cannot eavesdrop, however its trivial for someone | ||
| * to man-in-the-middle as we do not verify the server's authenticity. | ||
| * Please do not use this for security sensitive applications! | ||
| */ | ||
| return NetHttpFlags_TlsNoVerify; | ||
| } | ||
|
|
||
| static i32 fetch_run_origin(NetRest* rest, const String targetPath, const FetchOrigin* origin) { | ||
| i32 retCode = 0; | ||
|
|
||
| NetHttpAuth auth = {0}; | ||
| if (!string_is_empty(origin->authUser)) { | ||
| auth = (NetHttpAuth){ | ||
| .type = NetHttpAuthType_Basic, | ||
| .user = origin->authUser, | ||
| .pw = origin->authPass, | ||
| }; | ||
| } | ||
|
|
||
| NetRestId* requests = alloc_array_t(g_allocHeap, NetRestId, origin->assets.count); | ||
|
|
||
| // Start a GET request for all assets. | ||
| for (u32 i = 0; i != origin->assets.count; ++i) { | ||
| const String uri = fetch_config_uri_scratch(origin, origin->assets.values[i]); | ||
| requests[i] = net_rest_get(rest, origin->host, uri, &auth, null); | ||
| } | ||
|
|
||
| // Save the results. | ||
| for (u32 i = 0; i != origin->assets.count; ++i) { | ||
| const NetRestId request = requests[i]; | ||
| const String asset = origin->assets.values[i]; | ||
|
|
||
| // Wait for the request to be done. | ||
| while (!net_rest_done(rest, request)) { | ||
| thread_sleep(time_milliseconds(100)); | ||
| } | ||
|
|
||
| // Save the asset to disk. | ||
| const NetResult result = net_rest_result(rest, request); | ||
| if (result == NetResult_Success) { | ||
| const String path = path_build_scratch(targetPath, asset); | ||
| const String data = net_rest_data(rest, request); | ||
|
|
||
| FileResult saveRes = file_create_dir_sync(path_parent(path)); | ||
| if (saveRes == FileResult_Success) { | ||
| saveRes = file_write_to_path_atomic(path, data); | ||
| } | ||
| if (saveRes != FileResult_Success) { | ||
| log_e( | ||
| "Asset save failed: '{}'", | ||
| log_param("asset", fmt_text(asset)), | ||
| log_param("path", fmt_path(path)), | ||
| log_param("error", fmt_text(file_result_str(saveRes)))); | ||
| retCode = 2; | ||
| } else { | ||
| log_i( | ||
| "Asset fetched: '{}'", | ||
| log_param("asset", fmt_text(asset)), | ||
| log_param("size", fmt_size(data.size))); | ||
| } | ||
| } else { | ||
| log_e( | ||
| "Asset fetch failed: '{}'", | ||
| log_param("asset", fmt_text(asset)), | ||
| log_param("error", fmt_text(net_result_str(result)))); | ||
| retCode = 1; | ||
| } | ||
| net_rest_release(rest, request); | ||
| } | ||
|
|
||
| alloc_free_array_t(g_allocHeap, requests, origin->assets.count); | ||
| return retCode; | ||
| } | ||
|
|
||
| static i32 fetch_run(const String configPath) { | ||
| const TimeSteady timeStart = time_steady_clock(); | ||
|
|
||
| i32 retCode = 0; | ||
| NetRest* rest = null; | ||
| FetchConfig cfg; | ||
| if (!fetch_config_load(configPath, &cfg)) { | ||
| return 1; | ||
| } | ||
|
|
||
| DynString targetPath = dynstring_create(g_allocHeap, 128); | ||
| path_build(&targetPath, path_parent(configPath), cfg.targetPath); | ||
|
|
||
| const u32 maxOriginAssetCount = fetch_config_max_origin_assets(&cfg); | ||
| if (!maxOriginAssetCount) { | ||
| goto Done; | ||
| } | ||
| rest = net_rest_create(g_allocHeap, fetch_worker_count, maxOriginAssetCount, fetch_http_flags()); | ||
|
|
||
| heap_array_for_t(cfg.origins, FetchOrigin, origin) { | ||
| const i32 originRet = fetch_run_origin(rest, dynstring_view(&targetPath), origin); | ||
| retCode = math_max(retCode, originRet); | ||
| } | ||
|
|
||
| Done:; | ||
| const TimeDuration duration = time_steady_duration(timeStart, time_steady_clock()); | ||
| if (!retCode) { | ||
| log_i("Fetch finished", log_param("duration", fmt_duration(duration))); | ||
| } else { | ||
| log_e("Fetch failed", log_param("duration", fmt_duration(duration))); | ||
| } | ||
| if (rest) { | ||
| net_rest_destroy(rest); | ||
| } | ||
| fetch_config_destroy(&cfg); | ||
| dynstring_destroy(&targetPath); | ||
| return retCode; | ||
| } | ||
|
|
||
| static CliId g_optConfigPath, g_optVerbose, g_optHelp; | ||
|
|
||
| void app_cli_configure(CliApp* app) { | ||
| cli_app_register_desc(app, string_lit("Fetch utility.")); | ||
|
|
||
| g_optConfigPath = cli_register_arg(app, string_lit("config"), CliOptionFlags_Required); | ||
| cli_register_desc(app, g_optConfigPath, string_lit("Path to a fetch config file.")); | ||
| cli_register_validator(app, g_optConfigPath, cli_validate_file_regular); | ||
|
|
||
| g_optVerbose = cli_register_flag(app, 'v', string_lit("verbose"), CliOptionFlags_None); | ||
|
|
||
| g_optHelp = cli_register_flag(app, 'h', string_lit("help"), CliOptionFlags_None); | ||
| cli_register_desc(app, g_optHelp, string_lit("Display this help page.")); | ||
| cli_register_exclusions(app, g_optHelp, g_optConfigPath); | ||
| cli_register_exclusions(app, g_optHelp, g_optVerbose); | ||
| } | ||
|
|
||
| i32 app_cli_run(const CliApp* app, const CliInvocation* invoc) { | ||
| i32 retCode = 0; | ||
| if (cli_parse_provided(invoc, g_optHelp)) { | ||
| cli_help_write_file(app, g_fileStdOut); | ||
| return retCode; | ||
| } | ||
|
|
||
| const LogMask logMask = cli_parse_provided(invoc, g_optVerbose) ? LogMask_All : ~LogMask_Debug; | ||
| log_add_sink(g_logger, log_sink_pretty_default(g_allocHeap, logMask)); | ||
| log_add_sink(g_logger, log_sink_json_default(g_allocHeap, LogMask_All)); | ||
|
|
||
| fetch_data_init(); | ||
| net_init(); | ||
|
|
||
| const String configPath = cli_read_string(invoc, g_optConfigPath, string_empty); | ||
| retCode = fetch_run(configPath); | ||
|
|
||
| net_teardown(); | ||
| return retCode; | ||
| } |
Oops, something went wrong.