diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 35ae73e5d1f..bf94c24e286 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -19,7 +19,6 @@ in prefix, inlineHTML ? true, }: -settingsInfo: let @@ -27,11 +26,25 @@ let prefix: setting: { description, - documentDefault, - defaultValue, - aliases, - value, + experimentalFeature, + + # Whether we document the default, because it is machine agostic, + # or don't because because it is machine-specific. + documentDefault ? true, + + # The default value is JSON for new-style config, rather than then + # a string or boolean, for old-style config. + isJson ? false, + + defaultValue ? null, + + subSettings ? null, + + aliases ? [ ], + + # The current value for this setting. Purposefully unused. + value ? null, }: let result = squash '' @@ -50,7 +63,7 @@ let ${description} - **Default:** ${showDefault documentDefault defaultValue} + ${showDefaultOrSubSettings} ${showAliases aliases} ''; @@ -72,9 +85,24 @@ let > ``` ''; + showDefaultOrSubSettings = + if !isAttrs subSettings then + # No subsettings, instead single setting. Show the default value. + '' + **Default:** ${showDefault} + '' + else + # Indent the nested sub-settings, and append the outer setting name onto the prefix + indent " " '' + **Nullable sub-settings**: ${if subSettings.nullable then "true" else "false"} + ${builtins.trace prefix (showSettings "${prefix}-${setting}" subSettings.map)} + ''; + showDefault = - documentDefault: defaultValue: if documentDefault then + if isJson then + "`${builtins.toJSON defaultValue}`" + else # a StringMap value type is specified as a string, but # this shows the value type. The empty stringmap is `null` in # JSON, but that converts to `{ }` here. @@ -95,5 +123,7 @@ let in result; + showSettings = + prefix: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo)); in -concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo)) +showSettings prefix diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index b7b437e9c81..fee1c681bc9 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -43,7 +43,7 @@ Store * nix_store_open(nix_c_context * context, const char * uri, const char *** if (!params) return new Store{nix::openStore(uri_str)}; - nix::Store::Config::Params params_map; + nix::StoreReference::Params params_map; for (size_t i = 0; params[i] != nullptr; i++) { params_map[params[i][0]] = params[i][1]; } diff --git a/src/libstore-tests/data/store-reference/auto.json b/src/libstore-tests/data/store-reference/auto.json new file mode 100644 index 00000000000..da510f64949 --- /dev/null +++ b/src/libstore-tests/data/store-reference/auto.json @@ -0,0 +1,3 @@ +{ + "scheme": "auto" +} diff --git a/src/libstore-tests/data/store-reference/auto_param.json b/src/libstore-tests/data/store-reference/auto_param.json new file mode 100644 index 00000000000..41826c83b56 --- /dev/null +++ b/src/libstore-tests/data/store-reference/auto_param.json @@ -0,0 +1,4 @@ +{ + "root": "/foo/bar/baz", + "scheme": "auto" +} diff --git a/src/libstore-tests/data/store-reference/local_1.json b/src/libstore-tests/data/store-reference/local_1.json new file mode 100644 index 00000000000..3b44e382b7f --- /dev/null +++ b/src/libstore-tests/data/store-reference/local_1.json @@ -0,0 +1,5 @@ +{ + "authority": "", + "root": "/foo/bar/baz", + "scheme": "local" +} diff --git a/src/libstore-tests/data/store-reference/local_2.json b/src/libstore-tests/data/store-reference/local_2.json new file mode 100644 index 00000000000..346b0cf225f --- /dev/null +++ b/src/libstore-tests/data/store-reference/local_2.json @@ -0,0 +1,5 @@ +{ + "authority": "/foo/bar/baz", + "scheme": "local", + "trusted": true +} diff --git a/src/libstore-tests/data/store-reference/ssh.json b/src/libstore-tests/data/store-reference/ssh.json new file mode 100644 index 00000000000..e5b0b367ffd --- /dev/null +++ b/src/libstore-tests/data/store-reference/ssh.json @@ -0,0 +1,4 @@ +{ + "authority": "localhost", + "scheme": "ssh" +} diff --git a/src/libstore-tests/data/store-reference/unix.json b/src/libstore-tests/data/store-reference/unix.json new file mode 100644 index 00000000000..8943e959dd6 --- /dev/null +++ b/src/libstore-tests/data/store-reference/unix.json @@ -0,0 +1,6 @@ +{ + "authority": "", + "max-connections": 7, + "scheme": "unix", + "trusted": true +} diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc new file mode 100644 index 00000000000..655166d38c3 --- /dev/null +++ b/src/libstore-tests/dummy-store.cc @@ -0,0 +1,20 @@ +#include + +#include "nix/store/dummy-store.hh" +#include "nix/store/globals.hh" + +namespace nix { + +TEST(DummyStore, constructConfig) +{ + DummyStoreConfig config{"dummy", "", {}}; + + EXPECT_EQ(config.storeDir, settings.nixStore); +} + +TEST(DummyStore, constructConfigNoAuthority) +{ + EXPECT_THROW(DummyStoreConfig("dummy", "not-allowed", {}), UsageError); +} + +} // namespace nix diff --git a/src/libstore-tests/legacy-ssh-store.cc b/src/libstore-tests/legacy-ssh-store.cc index 158da2831ac..1e4a7f352fa 100644 --- a/src/libstore-tests/legacy-ssh-store.cc +++ b/src/libstore-tests/legacy-ssh-store.cc @@ -9,11 +9,13 @@ TEST(LegacySSHStore, constructConfig) LegacySSHStoreConfig config{ "ssh", "localhost", - StoreConfig::Params{ + StoreReference::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, }, }}; EXPECT_EQ( diff --git a/src/libstore-tests/local-overlay-store.cc b/src/libstore-tests/local-overlay-store.cc index fe064c3a51c..175e5d0f44e 100644 --- a/src/libstore-tests/local-overlay-store.cc +++ b/src/libstore-tests/local-overlay-store.cc @@ -1,9 +1,6 @@ -// FIXME: Odd failures for templates that are causing the PR to break -// for now with discussion with @Ericson2314 to comment out. -#if 0 -# include +#include -# include "nix/store/local-overlay-store.hh" +#include "nix/store/local-overlay-store.hh" namespace nix { @@ -31,4 +28,3 @@ TEST(LocalOverlayStore, constructConfig_rootPath) } } // namespace nix -#endif diff --git a/src/libstore-tests/local-store.cc b/src/libstore-tests/local-store.cc index ece277609ec..6d9009825a4 100644 --- a/src/libstore-tests/local-store.cc +++ b/src/libstore-tests/local-store.cc @@ -1,15 +1,6 @@ -// FIXME: Odd failures for templates that are causing the PR to break -// for now with discussion with @Ericson2314 to comment out. -#if 0 -# include +#include -# include "nix/store/local-store.hh" - -// Needed for template specialisations. This is not good! When we -// overhaul how store configs work, this should be fixed. -# include "nix/util/args.hh" -# include "nix/util/config-impl.hh" -# include "nix/util/abstract-setting-to-json.hh" +#include "nix/store/local-store.hh" namespace nix { @@ -37,4 +28,3 @@ TEST(LocalStore, constructConfig_rootPath) } } // namespace nix -#endif diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 8a1ff40f074..93867f0aa1f 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -58,6 +58,7 @@ sources = files( 'derivation-advanced-attrs.cc', 'derivation.cc', 'derived-path.cc', + 'dummy-store.cc', 'downstream-placeholder.cc', 'http-binary-cache-store.cc', 'legacy-ssh-store.cc', diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 4eb95360a6a..c5d35ccb6c6 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -97,7 +97,7 @@ TEST_F(nix_api_util_context, nix_store_open_dummy) nix_libstore_init(ctx); Store * store = nix_store_open(ctx, "dummy://", nullptr); ASSERT_EQ(NIX_OK, ctx->last_err_code); - ASSERT_STREQ("dummy", store->ptr->getUri().c_str()); + ASSERT_STREQ("dummy://", store->ptr->getUri().c_str()); std::string str; nix_store_get_version(ctx, store, OBSERVE_STRING(str)); diff --git a/src/libstore-tests/ssh-store.cc b/src/libstore-tests/ssh-store.cc index ccb87b767a9..4ab04251ee1 100644 --- a/src/libstore-tests/ssh-store.cc +++ b/src/libstore-tests/ssh-store.cc @@ -1,22 +1,21 @@ -// FIXME: Odd failures for templates that are causing the PR to break -// for now with discussion with @Ericson2314 to comment out. -#if 0 -# include +#include -# include "nix/store/ssh-store.hh" +#include "nix/store/ssh-store.hh" namespace nix { TEST(SSHStore, constructConfig) { SSHStoreConfig config{ - "ssh", + "ssh-ng", "localhost", - StoreConfig::Params{ + StoreReference::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, }, }, }; @@ -31,16 +30,64 @@ TEST(SSHStore, constructConfig) TEST(MountedSSHStore, constructConfig) { - MountedSSHStoreConfig config{ - "mounted-ssh", + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "mounted-ssh-store"); + + SSHStoreConfig config{ + "ssh-ng", + "localhost", + StoreReference::Params{ + { + "remote-program", + { + "foo", + "bar", + }, + }, + { + "mounted", + nlohmann::json::object_t{}, + }, + }, + mockXpSettings, + }; + + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); + + ASSERT_TRUE(config.mounted); + + EXPECT_EQ(config.mounted->realStoreDir, "/nix/store"); +} + +TEST(MountedSSHStore, constructConfigWithFunnyRealStoreDir) +{ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "mounted-ssh-store"); + + SSHStoreConfig config{ + "ssh-ng", "localhost", - StoreConfig::Params{ + StoreReference::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, + }, + { + "mounted", + nlohmann::json::object_t{ + {"real", "/foo/bar"}, + }, }, }, + mockXpSettings, }; EXPECT_EQ( @@ -49,7 +96,10 @@ TEST(MountedSSHStore, constructConfig) "foo", "bar", })); + + ASSERT_TRUE(config.mounted); + + EXPECT_EQ(config.mounted->realStoreDir, "/foo/bar"); } } -#endif diff --git a/src/libstore-tests/store-reference.cc b/src/libstore-tests/store-reference.cc index dd1b8309072..70d1eb18778 100644 --- a/src/libstore-tests/store-reference.cc +++ b/src/libstore-tests/store-reference.cc @@ -17,14 +17,14 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest std::filesystem::path goldenMaster(PathView testStem) const override { - return unitTestData / (testStem + ".txt"); + return unitTestData / testStem; } }; #define URI_TEST_READ(STEM, OBJ) \ TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_uri) \ { \ - readTest(#STEM, ([&](const auto & encoded) { \ + readTest(#STEM ".txt", ([&](const auto & encoded) { \ StoreReference expected = OBJ; \ auto got = StoreReference::parse(encoded); \ ASSERT_EQ(got, expected); \ @@ -35,7 +35,7 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_uri) \ { \ writeTest( \ - #STEM, \ + #STEM ".txt", \ [&]() -> StoreReference { return OBJ; }, \ [](const auto & file) { return StoreReference::parse(readFile(file)); }, \ [](const auto & file, const auto & got) { return writeFile(file, got.render()); }); \ @@ -45,14 +45,43 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest URI_TEST_READ(STEM, OBJ) \ URI_TEST_WRITE(STEM, OBJ) -URI_TEST( +#define JSON_TEST_READ(STEM, OBJ) \ + TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_json) \ + { \ + readTest(#STEM ".json", ([&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + StoreReference expected = OBJ; \ + StoreReference got = encoded; \ + ASSERT_EQ(got, expected); \ + })); \ + } + +#define JSON_TEST_WRITE(STEM, OBJ) \ + TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_json) \ + { \ + writeTest( \ + #STEM ".json", \ + [&]() -> StoreReference { return OBJ; }, \ + [](const auto & file) -> StoreReference { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, json(got).dump(2) + "\n"); }); \ + } + +#define JSON_TEST(STEM, OBJ) \ + JSON_TEST_READ(STEM, OBJ) \ + JSON_TEST_WRITE(STEM, OBJ) + +#define BOTH_FORMATS_TEST(STEM, OBJ) \ + URI_TEST(STEM, OBJ) \ + JSON_TEST(STEM, OBJ) + +BOTH_FORMATS_TEST( auto, (StoreReference{ .variant = StoreReference::Auto{}, .params = {}, })) -URI_TEST( +BOTH_FORMATS_TEST( auto_param, (StoreReference{ .variant = StoreReference::Auto{}, @@ -81,13 +110,13 @@ static StoreReference localExample_2{ }, .params = { - {"trusted", "true"}, + {"trusted", true}, }, }; -URI_TEST(local_1, localExample_1) +BOTH_FORMATS_TEST(local_1, localExample_1) -URI_TEST(local_2, localExample_2) +BOTH_FORMATS_TEST(local_2, localExample_2) URI_TEST_READ(local_shorthand_1, localExample_1) @@ -100,16 +129,16 @@ static StoreReference unixExample{ }, .params = { - {"max-connections", "7"}, - {"trusted", "true"}, + {"max-connections", 7}, + {"trusted", true}, }, }; -URI_TEST(unix, unixExample) +BOTH_FORMATS_TEST(unix, unixExample) URI_TEST_READ(unix_shorthand, unixExample) -URI_TEST( +BOTH_FORMATS_TEST( ssh, (StoreReference{ .variant = diff --git a/src/libstore-tests/uds-remote-store.cc b/src/libstore-tests/uds-remote-store.cc index c6a92666831..c215d6e18ff 100644 --- a/src/libstore-tests/uds-remote-store.cc +++ b/src/libstore-tests/uds-remote-store.cc @@ -1,9 +1,6 @@ -// FIXME: Odd failures for templates that are causing the PR to break -// for now with discussion with @Ericson2314 to comment out. -#if 0 -# include +#include -# include "nix/store/uds-remote-store.hh" +#include "nix/store/uds-remote-store.hh" namespace nix { @@ -20,4 +17,3 @@ TEST(UDSRemoteStore, constructConfigWrongScheme) } } // namespace nix -#endif diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4df9651f03f..c79b5c03b35 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -13,6 +13,7 @@ #include "nix/util/callback.hh" #include "nix/util/signals.hh" #include "nix/util/archive.hh" +#include "nix/store/config-parse-impl.hh" #include #include @@ -24,25 +25,111 @@ namespace nix { -BinaryCacheStore::BinaryCacheStore(Config & config) - : config{config} +constexpr static const BinaryCacheStoreConfigT binaryCacheStoreConfigDescriptions = { + .compression = { + .name = "compression", + .description = "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`).", + }, + .writeNARListing = { + .name = "write-nar-listing", + .description = "Whether to write a JSON file that lists the files in each NAR.", + }, + .writeDebugInfo = { + .name = "index-debug-info", + .description = R"( + Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to + fetch debug info on demand + )", + }, + .secretKeyFile{ + .name = "secret-key", + .description = "Path to the secret key used to sign the binary cache.", + }, + .localNarCache{ + .name = "local-nar-cache", + .description = "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`.", + }, + .parallelCompression{ + .name = "parallel-compression", + .description = "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`.", + }, + .compressionLevel{ + .name = "compression-level", + .description = R"( + The *preset level* to be used when compressing NARs. + The meaning and accepted values depend on the compression method selected. + `-1` specifies that the default compression level should be used. + )", + }, +}; + +#define BINARY_CACHE_STORE_CONFIG_FIELDS(X) \ + X(compression), \ + X(writeNARListing), \ + X(writeDebugInfo), \ + X(secretKeyFile), \ + X(secretKeyFiles), \ + X(localNarCache), \ + X(parallelCompression), \ + X(compressionLevel), + +MAKE_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS) + +static BinaryCacheStoreConfigT binaryCacheStoreConfigDefaults() +{ + return { + .compression = {"xz"}, + .writeNARListing = {false}, + .writeDebugInfo = {false}, + .secretKeyFile = {""}, + .secretKeyFiles = {{}}, + .localNarCache = {""}, + .parallelCompression = {false}, + .compressionLevel = {-1}, + }; +} + +MAKE_APPLY_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS) + +BinaryCacheStore::Config::BinaryCacheStoreConfig( + const Store::Config & storeConfig, + const StoreReference::Params & params) + : BinaryCacheStoreConfigT{binaryCacheStoreConfigApplyParse(params)} + , storeConfig{storeConfig} +{ +} + +config::SettingDescriptionMap BinaryCacheStoreConfig::descriptions() +{ + constexpr auto & descriptions = binaryCacheStoreConfigDescriptions; + auto defaults = binaryCacheStoreConfigDefaults(); + return { + BINARY_CACHE_STORE_CONFIG_FIELDS(DESCRIBE_ROW) + }; +} + +BinaryCacheStore::BinaryCacheStore(const Config & config) + : Store{config.storeConfig} + , config{config} { if (config.secretKeyFile != "") signers.push_back(std::make_unique( SecretKey { readFile(config.secretKeyFile) })); - if (config.secretKeyFiles != "") { - std::stringstream ss(config.secretKeyFiles); - Path keyPath; - while (std::getline(ss, keyPath, ',')) { - signers.push_back(std::make_unique( - SecretKey { readFile(keyPath) })); - } + for (auto & keyPath : config.secretKeyFiles.value) { + signers.push_back(std::make_unique( + SecretKey { readFile(keyPath) })); } StringSink sink; sink << narVersionMagic1; narMagic = sink.s; + + // Want to call this but cannot, because virtual function lookup is + // disabled in a constructor. It is thus left to instances to call + // it instead. + + //init(); } void BinaryCacheStore::init() @@ -61,9 +148,11 @@ void BinaryCacheStore::init() throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'", getUri(), value, storeDir); } else if (name == "WantMassQuery") { - config.wantMassQuery.setDefault(value == "1"); + resolvedSubstConfig.wantMassQuery.value = + config.storeConfig.wantMassQuery.value_or(value == "1"); } else if (name == "Priority") { - config.priority.setDefault(std::stoi(value)); + resolvedSubstConfig.priority.value = + config.storeConfig.priority.value_or(std::stoi(value)); } } } diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index bcaa11a9671..97c9e987682 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -2,9 +2,56 @@ #include "nix/store/common-ssh-store-config.hh" #include "nix/store/ssh.hh" +#include "nix/store/config-parse-impl.hh" namespace nix { +constexpr static const CommonSSHStoreConfigT commonSSHStoreConfigDescriptions = { + .sshKey{ + .name = "ssh-key", + .description = "Path to the SSH private key used to authenticate to the remote machine.", + }, + .sshPublicHostKey{ + .name = "base64-ssh-public-host-key", + .description = "The public host key of the remote machine.", + }, + .compress{ + .name = "compress", + .description = "Whether to enable SSH compression.", + }, + .remoteStore{ + .name = "remote-store", + .description = R"( + [Store URL](@docroot@/store/types/index.md#store-url-format) + to be used on the remote machine. The default is `auto` + (i.e. use the Nix daemon or `/nix/store` directly). + )", + }, +}; + +#define COMMON_SSH_STORE_CONFIG_FIELDS(X) X(sshKey), X(sshPublicHostKey), X(compress), X(remoteStore), + +MAKE_PARSE(CommonSSHStoreConfig, commonSSHStoreConfig, COMMON_SSH_STORE_CONFIG_FIELDS) + +static CommonSSHStoreConfigT commonSSHStoreConfigDefaults() +{ + return { + .sshKey = {""}, + .sshPublicHostKey = {""}, + .compress = {false}, + .remoteStore = {""}, + }; +} + +MAKE_APPLY_PARSE(CommonSSHStoreConfig, commonSSHStoreConfig, COMMON_SSH_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap CommonSSHStoreConfig::descriptions() +{ + constexpr auto & descriptions = commonSSHStoreConfigDescriptions; + auto defaults = commonSSHStoreConfigDefaults(); + return {COMMON_SSH_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}; +} + static std::string extractConnStr(std::string_view scheme, std::string_view _connStr) { if (_connStr.empty()) @@ -22,9 +69,10 @@ static std::string extractConnStr(std::string_view scheme, std::string_view _con return connStr; } -CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params) - : StoreConfig(params) - , host(extractConnStr(scheme, host)) +CommonSSHStoreConfig::CommonSSHStoreConfig( + std::string_view scheme, std::string_view host, const StoreReference::Params & params) + : CommonSSHStoreConfigT{commonSSHStoreConfigApplyParse(params)} + , host{extractConnStr(scheme, host)} { } diff --git a/src/libstore/config-parse.cc b/src/libstore/config-parse.cc new file mode 100644 index 00000000000..5de5ba97769 --- /dev/null +++ b/src/libstore/config-parse.cc @@ -0,0 +1,71 @@ +#include + +#include "nix/store/config-parse.hh" +#include "nix/util/json-utils.hh" +#include "nix/util/util.hh" + +namespace nix::config { + +}; + +namespace nlohmann { + +using namespace nix::config; + +SettingDescription adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return { + .description = getString(valueAt(obj, "description")), + .experimentalFeature = valueAt(obj, "experimentalFeature").get>(), + .info = [&]() -> decltype(SettingDescription::info) { + if (auto documentDefault = optionalValueAt(obj, "documentDefault")) { + return SettingDescription::Single{ + .defaultValue = *documentDefault ? (std::optional{valueAt(obj, "defaultValue")}) + : (std::optional{}), + }; + } else { + auto & subObj = getObject(valueAt(obj, "subSettings")); + return SettingDescription::Sub{ + .nullable = valueAt(subObj, "nullable"), + .map = valueAt(subObj, "map"), + }; + } + }(), + }; +} + +void adl_serializer::to_json(json & obj, SettingDescription sd) +{ + obj.emplace("description", sd.description); + // obj.emplace("aliases", sd.aliases); + obj.emplace("experimentalFeature", sd.experimentalFeature); + + std::visit( + overloaded{ + [&](const SettingDescription::Single & single) { + // Indicate the default value is JSON, rather than a legacy setting + // boolean or string. + // + // TODO remove if we no longer have the legacy setting system / the + // code handling doc rendering of the settings is decoupled. + obj.emplace("isJson", true); + + // Cannot just use `null` because the default value might itself be + // `null`. + obj.emplace("documentDefault", single.defaultValue.has_value()); + + if (single.defaultValue.has_value()) + obj.emplace("defaultValue", *single.defaultValue); + }, + [&](const SettingDescription::Sub & sub) { + json subJson; + subJson.emplace("nullable", sub.nullable); + subJson.emplace("map", sub.map); + obj.emplace("subSettings", std::move(subJson)); + }, + }, + sd.info); +} + +} diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 819c47babce..3d0c099ef8c 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -1,33 +1,24 @@ +#include "nix/store/dummy-store.hh" #include "nix/store/store-registration.hh" #include "nix/util/callback.hh" namespace nix { -struct DummyStoreConfig : public std::enable_shared_from_this, virtual StoreConfig { - using StoreConfig::StoreConfig; - - DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - { - if (!authority.empty()) - throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); - } - - static const std::string name() { return "Dummy Store"; } - - static std::string doc() - { - return - #include "dummy-store.md" - ; - } +DummyStoreConfig::DummyStoreConfig( + std::string_view scheme, std::string_view authority, const StoreReference::Params & params) + : StoreConfig{params} +{ + if (!authority.empty()) + throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); +} - static StringSet uriSchemes() { - return {"dummy"}; - } +std::string DummyStoreConfig::doc() +{ + return + #include "dummy-store.md" + ; +} - ref openStore() const override; -}; struct DummyStore : virtual Store { @@ -42,7 +33,7 @@ struct DummyStore : virtual Store std::string getUri() override { - return *Config::uriSchemes().begin(); + return *Config::uriSchemes().begin() + "://"; } void queryPathInfoUncached(const StorePath & path, diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 8080fcfddc5..1d2280a2d85 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -771,7 +771,7 @@ struct curlFileTransfer : public FileTransfer } #if NIX_WITH_S3_SUPPORT - std::tuple parseS3Uri(std::string uri) + std::tuple parseS3Uri(std::string uri) { auto [path, params] = splitUriAndParams(uri); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 2b591dda96e..f3093841414 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -7,6 +7,15 @@ namespace nix { +config::SettingDescriptionMap HttpBinaryCacheStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(BinaryCacheStoreConfig::descriptions()); + return ret; +} + + MakeError(UploadToHTTP, Error); @@ -22,9 +31,9 @@ StringSet HttpBinaryCacheStoreConfig::uriSchemes() HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( std::string_view scheme, std::string_view _cacheUri, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) + const StoreReference::Params & params) + : Store::Config{params} + , BinaryCacheStoreConfig{*this, params} , cacheUri( std::string { scheme } + "://" @@ -60,15 +69,16 @@ class HttpBinaryCacheStore : using Config = HttpBinaryCacheStoreConfig; - ref config; + ref config; - HttpBinaryCacheStore(ref config) + HttpBinaryCacheStore(ref config) : Store{*config} - // TODO it will actually mutate the configuration , BinaryCacheStore{*config} , config{config} { diskCache = getNarInfoDiskCache(); + + init(); } std::string getUri() override @@ -80,15 +90,18 @@ class HttpBinaryCacheStore : { // FIXME: do this lazily? if (auto cacheInfo = diskCache->upToDateCacheExists(config->cacheUri)) { - config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); - config->priority.setDefault(cacheInfo->priority); + resolvedSubstConfig.wantMassQuery.value = + config->storeConfig.wantMassQuery.value_or(cacheInfo->wantMassQuery); + resolvedSubstConfig.priority.value = + config->storeConfig.priority.value_or(cacheInfo->priority); } else { try { BinaryCacheStore::init(); } catch (UploadToHTTP &) { throw Error("'%s' does not appear to be a binary cache", config->cacheUri); } - diskCache->createCache(config->cacheUri, config->storeDir, config->wantMassQuery, config->priority); + diskCache->createCache( + config->cacheUri, storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority); } } @@ -232,10 +245,7 @@ class HttpBinaryCacheStore : ref HttpBinaryCacheStore::Config::openStore() const { - return make_ref(ref{ - // FIXME we shouldn't actually need a mutable config - std::const_pointer_cast(shared_from_this()) - }); + return make_ref(ref{shared_from_this()}); } static RegisterStoreImplementation regHttpBinaryCacheStore; diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 43f2cf690dc..1cfcc930dad 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -13,43 +13,29 @@ namespace nix { struct NarInfo; -struct BinaryCacheStoreConfig : virtual StoreConfig +template class F> +struct BinaryCacheStoreConfigT { - using StoreConfig::StoreConfig; - - const Setting compression{this, "xz", "compression", - "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; - - const Setting writeNARListing{this, false, "write-nar-listing", - "Whether to write a JSON file that lists the files in each NAR."}; - - const Setting writeDebugInfo{this, false, "index-debug-info", - R"( - Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to - fetch debug info on demand - )"}; - - const Setting secretKeyFile{this, "", "secret-key", - "Path to the secret key used to sign the binary cache."}; - - const Setting secretKeyFiles{this, "", "secret-keys", - "List of comma-separated paths to the secret keys used to sign the binary cache."}; + F compression; + F writeNARListing; + F writeDebugInfo; + F secretKeyFile; + F> secretKeyFiles; + F localNarCache; + F parallelCompression; + F compressionLevel; +}; - const Setting localNarCache{this, "", "local-nar-cache", - "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; +struct BinaryCacheStoreConfig : + BinaryCacheStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); - const Setting parallelCompression{this, false, "parallel-compression", - "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; + const Store::Config & storeConfig; - const Setting compressionLevel{this, -1, "compression-level", - R"( - The *preset level* to be used when compressing NARs. - The meaning and accepted values depend on the compression method selected. - `-1` specifies that the default compression level should be used. - )"}; + BinaryCacheStoreConfig(const Store::Config &, const StoreReference::Params &); }; - /** * @note subclasses must implement at least one of the two * virtual getFile() methods. @@ -60,11 +46,7 @@ struct BinaryCacheStore : { using Config = BinaryCacheStoreConfig; - /** - * Intentionally mutable because some things we update due to the - * cache's own (remote side) settings. - */ - Config & config; + const Config & config; private: std::vector> signers; @@ -76,7 +58,7 @@ protected: const std::string cacheInfoFile = "nix-cache-info"; - BinaryCacheStore(Config &); + BinaryCacheStore(const Config &); public: @@ -114,7 +96,11 @@ public: public: - virtual void init() override; + /** + * Perform any necessary effectful operation to make the store up and + * running + */ + virtual void init(); private: diff --git a/src/libstore/include/nix/store/common-ssh-store-config.hh b/src/libstore/include/nix/store/common-ssh-store-config.hh index 82a78f0755a..ea4fd9faad1 100644 --- a/src/libstore/include/nix/store/common-ssh-store-config.hh +++ b/src/libstore/include/nix/store/common-ssh-store-config.hh @@ -7,27 +7,27 @@ namespace nix { class SSHMaster; -struct CommonSSHStoreConfig : virtual StoreConfig +template class F> +struct CommonSSHStoreConfigT { - using StoreConfig::StoreConfig; - - CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params); - - const Setting sshKey{this, "", "ssh-key", - "Path to the SSH private key used to authenticate to the remote machine."}; - - const Setting sshPublicHostKey{this, "", "base64-ssh-public-host-key", - "The public host key of the remote machine."}; + F sshKey; + F sshPublicHostKey; + F compress; + F remoteStore; +}; - const Setting compress{this, false, "compress", - "Whether to enable SSH compression."}; +struct CommonSSHStoreConfig : CommonSSHStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); - const Setting remoteStore{this, "", "remote-store", - R"( - [Store URL](@docroot@/store/types/index.md#store-url-format) - to be used on the remote machine. The default is `auto` - (i.e. use the Nix daemon or `/nix/store` directly). - )"}; + /** + * @param scheme Note this isn't stored by this mix-in class, but + * just used for better error messages. + */ + CommonSSHStoreConfig( + std::string_view scheme, + std::string_view host, + const StoreReference::Params & params); /** * The `parseURL` function supports both IPv6 URIs as defined in diff --git a/src/libstore/include/nix/store/config-parse-impl.hh b/src/libstore/include/nix/store/config-parse-impl.hh new file mode 100644 index 00000000000..dce77d73dd3 --- /dev/null +++ b/src/libstore/include/nix/store/config-parse-impl.hh @@ -0,0 +1,69 @@ +#pragma once +///@file + +#include + +#include "nix/store/config-parse.hh" +#include "nix/util/util.hh" +#include "nix/util/configuration.hh" + +namespace nix::config { + +template +std::optional +SettingInfo::parseConfig(const nlohmann::json::object_t & map, const ExperimentalFeatureSettings & xpSettings) const +{ + const nlohmann::json * p = get(map, name); + if (p && experimentalFeature) + xpSettings.require(*experimentalFeature); + return p ? (std::optional{p->get()}) : std::nullopt; +} + +template +std::pair SettingInfo::describe(const PlainValue & def) const +{ + return { + std::string{name}, + SettingDescription{ + .description = stripIndentation(description), + .experimentalFeature = experimentalFeature, + .info = + SettingDescription::Single{ + .defaultValue = documentDefault ? (std::optional{nlohmann::json(def.value)}) + : (std::optional{}), + }, + }, + }; +} + +/** + * Look up the setting's name in a map, falling back on the default if + * it does not exist. + */ +#define CONFIG_ROW(FIELD) .FIELD = descriptions.FIELD.parseConfig(params, xpSettings) + +#define APPLY_ROW(FIELD) .FIELD = {.value = parsed.FIELD.value_or(std::move(defaults.FIELD))} + +#define DESCRIBE_ROW(FIELD) \ + { \ + descriptions.FIELD.describe(defaults.FIELD), \ + } + +#define MAKE_PARSE(CAPITAL, LOWER, FIELDS) \ + static CAPITAL##T LOWER##Parse( \ + const StoreReference::Params & params, \ + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) \ + { \ + constexpr auto & descriptions = LOWER##Descriptions; \ + return {FIELDS(CONFIG_ROW)}; \ + } + +#define MAKE_APPLY_PARSE(CAPITAL, LOWER, FIELDS) \ + static CAPITAL##T LOWER##ApplyParse(const StoreReference::Params & params) \ + { \ + auto defaults = LOWER##Defaults(); \ + auto parsed = LOWER##Parse(params); \ + return {FIELDS(APPLY_ROW)}; \ + } + +} diff --git a/src/libstore/include/nix/store/config-parse.hh b/src/libstore/include/nix/store/config-parse.hh new file mode 100644 index 00000000000..63966fbf334 --- /dev/null +++ b/src/libstore/include/nix/store/config-parse.hh @@ -0,0 +1,145 @@ +#pragma once +///@file + +#include + +#include "nix/util/config-abstract.hh" +#include "nix/util/json-impls.hh" +#include "nix/util/experimental-features.hh" + +namespace nix { + +struct ExperimentalFeatureSettings; + +}; + +namespace nix::config { + +struct SettingDescription; + +/** + * Typed version used as source of truth, and for operations like + * defaulting configurations. + * + * It is important that this type support `constexpr` values to avoid + * running into issues with static initialization order. + */ +template +struct SettingInfo +{ + /** + * Name of the setting, used when parsing configuration maps + */ + std::string_view name; + + /** + * Description of the setting. It is used just for documentation. + */ + std::string_view description; + +#if 0 + /** + * Other names of the setting also used when parsing configuration + * maps. This is useful for back-compat, etc. + */ + std::set aliases; +#endif + + /** + * `ExperimentalFeature` that must be enabled if the setting is + * allowed to be used + */ + std::optional experimentalFeature; + + /** + * Whether to document the default value. (Some defaults are + * system-specific and should not be documented.) + */ + bool documentDefault = true; + + /** + * Describe the setting as a key-value pair (name -> other info). + * The default value will be rendered to JSON if it is to be + * documented. + */ + std::pair describe(const PlainValue & def) const; + + std::optional + parseConfig(const nlohmann::json::object_t & map, const ExperimentalFeatureSettings & xpSettings) const; +}; + +struct SettingDescription; + +/** + * Map of setting names to descriptions of those settings. + */ +using SettingDescriptionMap = std::map; + +/** + * Untyped version used for rendering docs. This is not the source of + * truth, it is generated from the typed one. + * + * @note No `name` field because this is intended to be used as the value type + * of a map + */ +struct SettingDescription +{ + /** + * @see SettingInfo::description + */ + std::string description; + +#if 0 + /** + * @see SettingInfo::aliases + */ + StringSet aliases; +#endif + + /** + * @see SettingInfo::experimentalFeature + */ + std::optional experimentalFeature; + + /** + * A single leaf setting, to be optionally specified by arbitrary + * value (of some type) or left default. + */ + struct Single + { + /** + * Optional, for the `SettingInfo::documentDefault = false` case. + */ + std::optional defaultValue; + }; + + /** + * A nested settings object + */ + struct Sub + { + /** + * If `false`, this is just pure namespaceing. If `true`, we + * have a distinction between `null` and `{}`, meaning + * enabling/disabling the entire settings group. + */ + bool nullable = true; + + SettingDescriptionMap map; + }; + + /** + * Variant for `info` below + */ + using Info = std::variant; + + /** + * More information about this setting, depending on whether its the + * single leaf setting or subsettings case + */ + Info info; +}; + +} + +JSON_IMPL(config::SettingDescription) diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh new file mode 100644 index 00000000000..1c64e253909 --- /dev/null +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -0,0 +1,24 @@ +#include "nix/store/store-api.hh" + +namespace nix { + +struct DummyStoreConfig : std::enable_shared_from_this, StoreConfig +{ + DummyStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params); + + static const std::string name() + { + return "Dummy Store"; + } + + static std::string doc(); + + static StringSet uriSchemes() + { + return {"dummy"}; + } + + ref openStore() const override; +}; + +} diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index 66ec5f8d254..0c1999634ce 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -3,13 +3,13 @@ namespace nix { struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this, - virtual Store::Config, + Store::Config, BinaryCacheStoreConfig { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + static config::SettingDescriptionMap descriptions(); HttpBinaryCacheStoreConfig( - std::string_view scheme, std::string_view cacheUri, const Store::Config::Params & params); + std::string_view scheme, std::string_view cacheUri, const StoreReference::Params & params); Path cacheUri; diff --git a/src/libstore/include/nix/store/legacy-ssh-store.hh b/src/libstore/include/nix/store/legacy-ssh-store.hh index 65f29d6499d..142d8124fef 100644 --- a/src/libstore/include/nix/store/legacy-ssh-store.hh +++ b/src/libstore/include/nix/store/legacy-ssh-store.hh @@ -10,29 +10,31 @@ namespace nix { -struct LegacySSHStoreConfig : std::enable_shared_from_this, virtual CommonSSHStoreConfig +template class F> +struct LegacySSHStoreConfigT { - using CommonSSHStoreConfig::CommonSSHStoreConfig; + F remoteProgram; + F maxConnections; +}; + +struct LegacySSHStoreConfig : + std::enable_shared_from_this, + Store::Config, + CommonSSHStoreConfig, + LegacySSHStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); + + /** + * Hack for getting remote build log output. Intentionally not a + * documented user-visible setting. + */ + Descriptor logFD = INVALID_DESCRIPTOR; LegacySSHStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params); - -#ifndef _WIN32 - // Hack for getting remote build log output. - // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in - // the documentation - const Setting logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"}; -#else - Descriptor logFD = INVALID_DESCRIPTOR; -#endif - - const Setting remoteProgram{this, {"nix-store"}, "remote-program", - "Path to the `nix-store` executable on the remote machine."}; - - const Setting maxConnections{this, 1, "max-connections", - "Maximum number of concurrent SSH connections."}; + const StoreReference::Params & params); /** * Hack for hydra diff --git a/src/libstore/include/nix/store/local-binary-cache-store.hh b/src/libstore/include/nix/store/local-binary-cache-store.hh index 780eaf4808e..fb277250ff5 100644 --- a/src/libstore/include/nix/store/local-binary-cache-store.hh +++ b/src/libstore/include/nix/store/local-binary-cache-store.hh @@ -3,16 +3,17 @@ namespace nix { struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this, - virtual Store::Config, + Store::Config, BinaryCacheStoreConfig { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + static config::SettingDescriptionMap descriptions(); /** * @param binaryCacheDir `file://` is a short-hand for `file:///` * for now. */ - LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const Params & params); + LocalBinaryCacheStoreConfig( + std::string_view scheme, PathView binaryCacheDir, const StoreReference::Params & params); Path binaryCacheDir; diff --git a/src/libstore/include/nix/store/local-fs-store.hh b/src/libstore/include/nix/store/local-fs-store.hh index f9421b7febc..ca320d567e5 100644 --- a/src/libstore/include/nix/store/local-fs-store.hh +++ b/src/libstore/include/nix/store/local-fs-store.hh @@ -7,9 +7,24 @@ namespace nix { -struct LocalFSStoreConfig : virtual StoreConfig +template class F> +struct LocalFSStoreConfigT { - using StoreConfig::StoreConfig; + F> rootDir; + F stateDir; + F logDir; + F realStoreDir; +}; + +struct LocalFSStoreConfig : LocalFSStoreConfigT +{ + const Store::Config & storeConfig; + + static config::SettingDescriptionMap descriptions(); + + LocalFSStoreConfig( + const Store::Config & storeConfig, + const StoreReference::Params &); /** * Used to override the `root` settings. Can't be done via modifying @@ -18,25 +33,10 @@ struct LocalFSStoreConfig : virtual StoreConfig * * @todo Make this less error-prone with new store settings system. */ - LocalFSStoreConfig(PathView path, const Params & params); - - OptionalPathSetting rootDir{this, std::nullopt, - "root", - "Directory prefixed to all other paths."}; - - PathSetting stateDir{this, - rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir, - "state", - "Directory where Nix will store state."}; - - PathSetting logDir{this, - rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir, - "log", - "directory where Nix will store log files."}; - - PathSetting realStoreDir{this, - rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", - "Physical path of the Nix store."}; + LocalFSStoreConfig( + const Store::Config & storeConfig, + PathView path, + const StoreReference::Params & params); }; struct LocalFSStore : diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index 6077d9e535c..1ef5c8bc249 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -2,59 +2,29 @@ namespace nix { + +template class F> +struct LocalOverlayStoreConfigT +{ + F> lowerStoreConfig; + F upperLayer; + F checkMount; + F remountHook; +}; + /** * Configuration for `LocalOverlayStore`. */ -struct LocalOverlayStoreConfig : virtual LocalStoreConfig +struct LocalOverlayStoreConfig : + LocalStoreConfig, + LocalOverlayStoreConfigT { - LocalOverlayStoreConfig(const StringMap & params) - : LocalOverlayStoreConfig("local-overlay", "", params) - { } - - LocalOverlayStoreConfig(std::string_view scheme, PathView path, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(path, params) - , LocalStoreConfig(scheme, path, params) - { - } + static config::SettingDescriptionMap descriptions(); - const Setting lowerStoreUri{(StoreConfig*) this, "", "lower-store", - R"( - [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) - for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). - - Must be a store with a store dir on the file system. - Must be used as OverlayFS lower layer for this store's store dir. - )"}; - - const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer", - R"( - Directory containing the OverlayFS upper layer for this store's store dir. - )"}; - - Setting checkMount{(StoreConfig*) this, true, "check-mount", - R"( - Check that the overlay filesystem is correctly mounted. - - Nix does not manage the overlayfs mount point itself, but the correct - functioning of the overlay store does depend on this mount point being set up - correctly. Rather than just assume this is the case, check that the lowerdir - and upperdir options are what we expect them to be. This check is on by - default, but can be disabled if needed. - )"}; - - const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook", - R"( - Script or other executable to run when overlay filesystem needs remounting. - - This is occasionally necessary when deleting a store path that exists in both upper and lower layers. - In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly - is the only way to perform the deletion without creating a "whiteout". - However this causes the OverlayFS kernel data structures to get out-of-sync, - and can lead to 'stale file handle' errors; remounting solves the problem. - - The store directory is passed as an argument to the invoked executable. - )"}; + LocalOverlayStoreConfig( + std::string_view scheme, + PathView path, + const StoreReference::Params & params); static const std::string name() { return "Experimental Local Overlay Store"; } diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index efc59dc8cb7..00d3a2e7ee4 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -34,36 +34,34 @@ struct OptimiseStats uint64_t bytesFreed = 0; }; -struct LocalStoreConfig : std::enable_shared_from_this, virtual LocalFSStoreConfig +template class F> +struct LocalStoreConfigT { - using LocalFSStoreConfig::LocalFSStoreConfig; + F requireSigs; + F readOnly; +}; + +struct LocalStoreConfig : + std::enable_shared_from_this, + Store::Config, + LocalFSStore::Config, + LocalStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); + + LocalStoreConfig(const StoreReference::Params & params) + : LocalStoreConfig{"local", "", params} + {} LocalStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params); - - Setting requireSigs{this, - settings.requireSigs, - "require-sigs", - "Whether store paths copied into this store should have a trusted signature."}; + const StoreReference::Params & params); - Setting readOnly{this, - false, - "read-only", - R"( - Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. - - Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. - - Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. - - > **Warning** - > Do not use this unless the filesystem is read-only. - > - > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. - > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. - )"}; + /** + * For `RestrictedStore` + */ + LocalStoreConfig(const LocalStoreConfig &); static const std::string name() { return "Local Store"; } diff --git a/src/libstore/include/nix/store/machines.hh b/src/libstore/include/nix/store/machines.hh index 2bf7408f624..df7cba406b7 100644 --- a/src/libstore/include/nix/store/machines.hh +++ b/src/libstore/include/nix/store/machines.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + #include "nix/util/ref.hh" #include "nix/store/store-reference.hh" diff --git a/src/libstore/include/nix/store/meson.build b/src/libstore/include/nix/store/meson.build index a1843041760..a6944173b74 100644 --- a/src/libstore/include/nix/store/meson.build +++ b/src/libstore/include/nix/store/meson.build @@ -24,13 +24,16 @@ headers = [config_pub_h] + files( 'common-protocol-impl.hh', 'common-protocol.hh', 'common-ssh-store-config.hh', + 'config-parse-impl.hh', + 'config-parse.hh', 'content-address.hh', 'daemon.hh', - 'derivations.hh', 'derivation-options.hh', + 'derivations.hh', 'derived-path-map.hh', 'derived-path.hh', 'downstream-placeholder.hh', + 'dummy-store.hh', 'filetransfer.hh', 'gc-store.hh', 'globals.hh', diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index dd2396fe32b..2a2f79083f3 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -18,17 +18,20 @@ struct FdSink; struct FdSource; template class Pool; -struct RemoteStoreConfig : virtual StoreConfig +template class F> +struct RemoteStoreConfigT { - using StoreConfig::StoreConfig; + F maxConnections; + F maxConnectionAge; +}; + +struct RemoteStoreConfig : RemoteStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); - const Setting maxConnections{this, 1, "max-connections", - "Maximum number of concurrent connections to the Nix daemon."}; + const Store::Config & storeConfig; - const Setting maxConnectionAge{this, - std::numeric_limits::max(), - "max-connection-age", - "Maximum age of a connection before it is closed."}; + RemoteStoreConfig(const Store::Config &, const StoreReference::Params &); }; /** diff --git a/src/libstore/include/nix/store/s3-binary-cache-store.hh b/src/libstore/include/nix/store/s3-binary-cache-store.hh index 9a123602e41..303a0dad00c 100644 --- a/src/libstore/include/nix/store/s3-binary-cache-store.hh +++ b/src/libstore/include/nix/store/s3-binary-cache-store.hh @@ -11,89 +11,33 @@ namespace nix { -struct S3BinaryCacheStoreConfig : std::enable_shared_from_this, virtual BinaryCacheStoreConfig +template class F> +struct S3BinaryCacheStoreConfigT { + F profile; + F region; + F scheme; + F endpoint; + F narinfoCompression; + F lsCompression; + F logCompression; + F multipartUpload; + F bufferSize; +}; + +struct S3BinaryCacheStoreConfig : std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig, + S3BinaryCacheStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); + + S3BinaryCacheStoreConfig( + std::string_view uriScheme, std::string_view bucketName, const StoreReference::Params & params); + std::string bucketName; - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - - S3BinaryCacheStoreConfig(std::string_view uriScheme, std::string_view bucketName, const Params & params); - - const Setting profile{ - this, - "", - "profile", - R"( - The name of the AWS configuration profile to use. By default - Nix will use the `default` profile. - )"}; - -protected: - - constexpr static const char * defaultRegion = "us-east-1"; - -public: - - const Setting region{ - this, - defaultRegion, - "region", - R"( - The region of the S3 bucket. If your bucket is not in - `us–east-1`, you should always explicitly specify the region - parameter. - )"}; - - const Setting scheme{ - this, - "", - "scheme", - R"( - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - )"}; - - const Setting endpoint{ - this, - "", - "endpoint", - R"( - The URL of the endpoint of an S3-compatible service such as MinIO. - Do not specify this setting if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and will use path-based - > addressing instead of virtual host based addressing. - )"}; - - const Setting narinfoCompression{ - this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; - - const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; - - const Setting logCompression{ - this, - "", - "log-compression", - R"( - Compression method for `log/*` files. It is recommended to - use a compression method supported by most web browsers - (e.g. `brotli`). - )"}; - - const Setting multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."}; - - const Setting bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; - - static const std::string name() + static std::string name() { return "S3 Binary Cache Store"; } @@ -112,9 +56,9 @@ struct S3BinaryCacheStore : virtual BinaryCacheStore { using Config = S3BinaryCacheStoreConfig; - ref config; + ref config; - S3BinaryCacheStore(ref); + S3BinaryCacheStore(ref); struct Stats { diff --git a/src/libstore/include/nix/store/ssh-store.hh b/src/libstore/include/nix/store/ssh-store.hh index fde165445fa..2045cb11b35 100644 --- a/src/libstore/include/nix/store/ssh-store.hh +++ b/src/libstore/include/nix/store/ssh-store.hh @@ -8,17 +8,27 @@ namespace nix { +template class F> +struct SSHStoreConfigT +{ + F remoteProgram; +}; + struct SSHStoreConfig : std::enable_shared_from_this, - virtual RemoteStoreConfig, - virtual CommonSSHStoreConfig + Store::Config, + RemoteStore::Config, + CommonSSHStoreConfig, + SSHStoreConfigT { - using CommonSSHStoreConfig::CommonSSHStoreConfig; - using RemoteStoreConfig::RemoteStoreConfig; + static config::SettingDescriptionMap descriptions(); - SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); + std::optional mounted; - const Setting remoteProgram{ - this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; + SSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreReference::Params & params, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); static const std::string name() { @@ -35,29 +45,4 @@ struct SSHStoreConfig : std::enable_shared_from_this, ref openStore() const override; }; -struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig -{ - MountedSSHStoreConfig(StringMap params); - MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params); - - static const std::string name() - { - return "Experimental SSH Store with filesystem mounted"; - } - - static StringSet uriSchemes() - { - return {"mounted-ssh-ng"}; - } - - static std::string doc(); - - static std::optional experimentalFeature() - { - return ExperimentalFeature::MountedSSHStore; - } - - ref openStore() const override; -}; - } diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 1648b13c1b2..c4d710c8c1c 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -9,7 +9,6 @@ #include "nix/util/lru-cache.hh" #include "nix/util/sync.hh" #include "nix/store/globals.hh" -#include "nix/util/configuration.hh" #include "nix/store/path-info.hh" #include "nix/util/repair-flag.hh" #include "nix/store/store-dir-config.hh" @@ -26,6 +25,32 @@ namespace nix { +/** + * About the class hierarchy of the store types: + * + * Each store type `Foo` consists of two classes: + * + * 1. A class `FooConfig : virtual StoreConfig` that contains the configuration + * for the store + * + * It should only contain members of type `const Setting` (or subclasses + * of it) and inherit the constructors of `StoreConfig` + * (`using StoreConfig::StoreConfig`). + * + * 2. A class `Foo : virtual Store, virtual FooConfig` that contains the + * implementation of the store. + * + * This class is expected to have a constructor `Foo(const StoreReference::Params & params)` + * that calls `StoreConfig(params)` (otherwise you're gonna encounter an + * `assertion failure` when trying to instantiate it). + * + * You can then register the new store using: + * + * ``` + * cpp static RegisterStoreImplementation regStore; + * ``` + */ + MakeError(SubstError, Error); /** * denotes a permanent build failure @@ -71,39 +96,43 @@ struct KeyedBuildResult; typedef std::map> StorePathCAMap; +template class F> +struct StoreConfigT +{ + F pathInfoCacheSize; + F isTrusted; + F systemFeatures; +}; + +template class F> +struct SubstituterConfigT +{ + F priority; + F wantMassQuery; +}; /** - * About the class hierarchy of the store types: - * - * Each store type `Foo` consists of two classes: - * - * 1. A class `FooConfig : virtual StoreConfig` that contains the configuration - * for the store - * - * It should only contain members of type `Setting` (or subclasses - * of it) and inherit the constructors of `StoreConfig` - * (`using StoreConfig::StoreConfig`). - * - * 2. A class `Foo : virtual Store` that contains the - * implementation of the store. - * - * This class is expected to have: - * - * 1. an alias `using Config = FooConfig;` - * - * 2. a constructor `Foo(ref params)`. - * - * You can then register the new store using: - * - * ``` - * cpp static RegisterStoreImplementation regStore; - * ``` + * @note In other cases we don't expose this function directly, but in + * this case we must because of `Store::resolvedSubstConfig` below. As + * the docs of that field describe, this is a case where the + * configuration is intentionally stateful. */ -struct StoreConfig : public StoreDirConfig +SubstituterConfigT substituterConfigDefaults(); + +/** + * @note `std::optional` rather than `config::PlainValue` is applied to + * `SubstitutorConfigT` because these are overrides. Caches themselves (not our + * config) can update default settings, but aren't allowed to update settings + * specified by the client (i.e. us). + */ +struct StoreConfig : + StoreDirConfig, + StoreConfigT, + SubstituterConfigT { - using StoreDirConfig::StoreDirConfig; + static config::SettingDescriptionMap descriptions(); - StoreConfig() = delete; + StoreConfig(const StoreReference::Params &); virtual ~StoreConfig() { } @@ -126,39 +155,6 @@ struct StoreConfig : public StoreDirConfig return std::nullopt; } - Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", - "Size of the in-memory store path metadata cache."}; - - Setting isTrusted{this, false, "trusted", - R"( - Whether paths from this store can be used as substitutes - even if they are not signed by a key listed in the - [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) - setting. - )"}; - - Setting priority{this, 0, "priority", - R"( - Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). - A lower value means a higher priority. - )"}; - - Setting wantMassQuery{this, false, "want-mass-query", - R"( - Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). - )"}; - - Setting systemFeatures{this, getDefaultSystemFeatures(), - "system-features", - R"( - Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. - - Example: `"kvm"` - )", - {}, - // Don't document the machine-specific default value - false}; - /** * Open a store of the type corresponding to this configuration * type. @@ -190,6 +186,14 @@ public: */ operator const Config &() const { return config; } + /** + * Resolved substituter configuration. This is intentionally mutable + * as store clients may do IO to ask the underlying store for their + * default setting values if the client config did not statically + * override them. + */ + SubstituterConfigT resolvedSubstConfig = substituterConfigDefaults(); + protected: struct PathInfoCacheValue { @@ -231,11 +235,6 @@ protected: Store(const Store::Config & config); public: - /** - * Perform any necessary effectful operation to make the store up and - * running - */ - virtual void init() {}; virtual ~Store() { } @@ -909,3 +908,6 @@ std::map drvOutputReferences( Store * evalStore = nullptr); } + +// Parses a Store URL, uses global state not pure so think about this +JSON_IMPL(ref) diff --git a/src/libstore/include/nix/store/store-dir-config.hh b/src/libstore/include/nix/store/store-dir-config.hh index ea0161d79c9..7d278a99dc7 100644 --- a/src/libstore/include/nix/store/store-dir-config.hh +++ b/src/libstore/include/nix/store/store-dir-config.hh @@ -3,8 +3,8 @@ #include "nix/store/path.hh" #include "nix/util/hash.hh" #include "nix/store/content-address.hh" -#include "nix/store/globals.hh" -#include "nix/util/configuration.hh" +#include "nix/store/store-reference.hh" +#include "nix/store/config-parse.hh" #include #include @@ -18,6 +18,20 @@ struct SourcePath; MakeError(BadStorePath, Error); MakeError(BadStorePathName, BadStorePath); +/** + * Underlying store directory configuration type. + * + * Don't worry to much about the `F` parameter, it just some abstract + * nonsense for the "higher-kinded data" pattern. It is used in each + * settings record in order to ensure don't forgot to parse or document + * settings field. + */ +template class F> +struct StoreDirConfigT +{ + F _storeDir; +}; + /** * @todo This should just be part of `StoreDirConfig`. However, it would * be a huge amount of churn if `Store` didn't have these methods @@ -31,8 +45,6 @@ struct MixStoreDirMethods { const Path & storeDir; - // pure methods - StorePath parseStorePath(std::string_view path) const; std::optional maybeParseStorePath(std::string_view path) const; @@ -101,35 +113,22 @@ struct MixStoreDirMethods }; /** - * Need to make this a separate class so I can get the right - * initialization order in the constructor for `StoreDirConfig`. - */ -struct StoreDirConfigBase : Config -{ - using Config::Config; - - const PathSetting storeDir_{this, settings.nixStore, - "store", - R"( - Logical location of the Nix store, usually - `/nix/store`. Note that you can only copy store paths - between stores if they have the same `store` setting. - )"}; -}; - -/** - * The order of `StoreDirConfigBase` and then `MixStoreDirMethods` is - * very important. This ensures that `StoreDirConfigBase::storeDir_` - * is initialized before we have our one chance (because references are - * immutable) to initialize `MixStoreDirMethods::storeDir`. + * Store directory configuration type. + * + * Combines the underlying `*T` type (with plain values for the fields) + * and the methods. + * + * The order of `StoreDirConfigT` and then + * `MixStoreDirMethods` is very important. This ensures that + * `StoreDirConfigT::storeDir_` is initialized + * before we have our one chance (because references are immutable) to + * initialize `MixStoreDirMethods::storeDir`. */ -struct StoreDirConfig : StoreDirConfigBase, MixStoreDirMethods +struct StoreDirConfig : StoreDirConfigT, MixStoreDirMethods { - using Params = StringMap; - - StoreDirConfig(const Params & params); + static config::SettingDescriptionMap descriptions(); - StoreDirConfig() = delete; + StoreDirConfig(const StoreReference::Params & params); virtual ~StoreDirConfig() = default; }; diff --git a/src/libstore/include/nix/store/store-reference.hh b/src/libstore/include/nix/store/store-reference.hh index 750bd827510..7a5f48c1493 100644 --- a/src/libstore/include/nix/store/store-reference.hh +++ b/src/libstore/include/nix/store/store-reference.hh @@ -2,8 +2,10 @@ ///@file #include +#include #include "nix/util/types.hh" +#include "nix/util/json-impls.hh" namespace nix { @@ -41,7 +43,17 @@ namespace nix { */ struct StoreReference { - using Params = StringMap; + /** + * Would do + * + * ``` + * using Params = nlohmann::json::object_t; + * ``` + * + * but cannot because `` doesn't have that. + * + */ + using Params = std::map>; /** * Special store reference `""` or `"auto"` @@ -70,7 +82,7 @@ struct StoreReference Params params; - bool operator==(const StoreReference & rhs) const = default; + bool operator==(const StoreReference & rhs) const; /** * Render the whole store reference as a URI, including parameters. @@ -89,3 +101,5 @@ struct StoreReference std::pair splitUriAndParams(const std::string & uri); } + +JSON_IMPL(StoreReference) diff --git a/src/libstore/include/nix/store/store-registration.hh b/src/libstore/include/nix/store/store-registration.hh index 3f82ff51ceb..deae9a54c09 100644 --- a/src/libstore/include/nix/store/store-registration.hh +++ b/src/libstore/include/nix/store/store-registration.hh @@ -29,6 +29,19 @@ struct StoreFactory */ StringSet uriSchemes; + /** + * @note This is a functional pointer for now because this situation: + * + * - We register store types with global initializers + * + * - The default values for some settings maybe depend on the settings globals. + * + * And because the ordering of global initialization is arbitrary, + * this is not allowed. For now, we can simply defer actually + * creating these maps until we need to later. + */ + config::SettingDescriptionMap (*configDescriptions)(); + /** * An experimental feature this type store is gated, if it is to be * experimental. @@ -40,21 +53,21 @@ struct StoreFactory * whatever comes after `://` and before `?`. */ std::function( - std::string_view scheme, std::string_view authorityPath, const Store::Config::Params & params)> + std::string_view scheme, std::string_view authorityPath, const StoreReference::Params & params)> parseConfig; - - /** - * Just for dumping the defaults. Kind of awkward this exists, - * because it means we cannot require fields to be manually - * specified so easily. - */ - std::function()> getConfig; }; struct Implementations { +private: + + /** + * The name of this type of store, and a factory for it. + */ using Map = std::map; +public: + static Map & registered(); template @@ -63,11 +76,11 @@ struct Implementations StoreFactory factory{ .doc = TConfig::doc(), .uriSchemes = TConfig::uriSchemes(), + .configDescriptions = TConfig::descriptions, .experimentalFeature = TConfig::experimentalFeature(), .parseConfig = ([](auto scheme, auto uri, auto & params) -> ref { return make_ref(scheme, uri, params); }), - .getConfig = ([]() -> ref { return make_ref(Store::Config::Params{}); }), }; auto [it, didInsert] = registered().insert({TConfig::name(), std::move(factory)}); if (!didInsert) { diff --git a/src/libstore/include/nix/store/uds-remote-store.hh b/src/libstore/include/nix/store/uds-remote-store.hh index e9793a9ee55..06df661c6ec 100644 --- a/src/libstore/include/nix/store/uds-remote-store.hh +++ b/src/libstore/include/nix/store/uds-remote-store.hh @@ -9,13 +9,15 @@ namespace nix { struct UDSRemoteStoreConfig : std::enable_shared_from_this, - virtual LocalFSStoreConfig, - virtual RemoteStoreConfig + Store::Config, + LocalFSStore::Config, + RemoteStore::Config { - // TODO(fzakaria): Delete this constructor once moved over to the factory pattern - // outlined in https://github.com/NixOS/nix/issues/10766 - using LocalFSStoreConfig::LocalFSStoreConfig; - using RemoteStoreConfig::RemoteStoreConfig; + static config::SettingDescriptionMap descriptions(); + + UDSRemoteStoreConfig(const StoreReference::Params & params) + : UDSRemoteStoreConfig{"unix", "", params} + {} /** * @param authority is the socket path. @@ -23,9 +25,7 @@ struct UDSRemoteStoreConfig : UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params); - - UDSRemoteStoreConfig(const Params & params); + const StoreReference::Params & params); static const std::string name() { return "Local Daemon Store"; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 9ec9e6eec19..5a34e48fc6f 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -12,19 +12,76 @@ #include "nix/store/ssh.hh" #include "nix/store/derivations.hh" #include "nix/util/callback.hh" +#include "nix/store/config-parse-impl.hh" #include "nix/store/store-registration.hh" namespace nix { -LegacySSHStoreConfig::LegacySSHStoreConfig( +constexpr static const LegacySSHStoreConfigT legacySSHStoreConfigDescriptions = { + .remoteProgram{ + .name = "remote-program", + .description = "Path to the `nix-store` executable on the remote machine.", + }, + .maxConnections{ + .name = "max-connections", + .description = "Maximum number of concurrent SSH connections.", + }, +}; + + +#define LEGACY_SSH_STORE_CONFIG_FIELDS(X) \ + X(remoteProgram), \ + X(maxConnections) + + +MAKE_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS) + + +static LegacySSHStoreConfigT legacySSHStoreConfigDefaults() +{ + return { + .remoteProgram = {{"nix-store"}}, + .maxConnections = {1}, + }; +} + + +MAKE_APPLY_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS) + + +config::SettingDescriptionMap LegacySSHStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(CommonSSHStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + { + constexpr auto & descriptions = legacySSHStoreConfigDescriptions; + auto defaults = legacySSHStoreConfigDefaults(); + ret.merge(decltype(ret){ + LEGACY_SSH_STORE_CONFIG_FIELDS(DESCRIBE_ROW) + }); + } + return ret; +} + + +LegacySSHStore::Config::LegacySSHStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params) - : StoreConfig(params) - , CommonSSHStoreConfig(scheme, authority, params) + const StoreReference::Params & params) + : Store::Config{params} + , CommonSSHStoreConfig{scheme, authority, params} + , LegacySSHStoreConfigT{legacySSHStoreConfigApplyParse(params)} { +#ifndef _WIN32 + if (auto * p = get(params, "log-fd")) { + logFD = p->get(); + } +#endif } + std::string LegacySSHStoreConfig::doc() { return diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 2f23135fae7..49cda491b5a 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -8,12 +8,21 @@ namespace nix { +config::SettingDescriptionMap LocalBinaryCacheStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(BinaryCacheStoreConfig::descriptions()); + return ret; +} + + LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( std::string_view scheme, PathView binaryCacheDir, const StoreReference::Params & params) : Store::Config{params} - , BinaryCacheStoreConfig{params} + , BinaryCacheStoreConfig{*this, params} , binaryCacheDir(binaryCacheDir) { } @@ -32,9 +41,9 @@ struct LocalBinaryCacheStore : { using Config = LocalBinaryCacheStoreConfig; - ref config; + ref config; - LocalBinaryCacheStore(ref config) + LocalBinaryCacheStore(ref config) : Store{*config} , BinaryCacheStore{*config} , config{config} @@ -126,10 +135,7 @@ StringSet LocalBinaryCacheStoreConfig::uriSchemes() } ref LocalBinaryCacheStoreConfig::openStore() const { - return make_ref(ref{ - // FIXME we shouldn't actually need a mutable config - std::const_pointer_cast(shared_from_this()) - }); + return make_ref(ref{shared_from_this()}); } static RegisterStoreImplementation regLocalBinaryCacheStore; diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index add3b04d237..73662f83272 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,3 +1,4 @@ +#include "nix/util/json-utils.hh" #include "nix/util/archive.hh" #include "nix/util/posix-source-accessor.hh" #include "nix/store/store-api.hh" @@ -5,20 +6,102 @@ #include "nix/store/globals.hh" #include "nix/util/compression.hh" #include "nix/store/derivations.hh" +#include "nix/store/config-parse-impl.hh" namespace nix { -LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params) - : StoreConfig(params) - // Default `?root` from `rootDir` if non set - // FIXME don't duplicate description once we don't have root setting - , rootDir{ - this, - !rootDir.empty() && params.count("root") == 0 - ? (std::optional{rootDir}) - : std::nullopt, - "root", - "Directory prefixed to all other paths."} +constexpr static const LocalFSStoreConfigT localFSStoreConfigDescriptions = { + .rootDir = { + .name = "root", + .description = "Directory prefixed to all other paths.", + }, + .stateDir = { + .name = "state", + .description = "Directory where Nix will store state.", + }, + .logDir = { + .name = "log", + .description = "directory where Nix will store log files.", + }, + .realStoreDir{ + .name = "real", + .description = "Physical path of the Nix store.", + }, +}; + +#define LOCAL_FS_STORE_CONFIG_FIELDS(X) \ + X(rootDir), \ + X(stateDir), \ + X(logDir), \ + X(realStoreDir), + +MAKE_PARSE(LocalFSStoreConfig, localFSStoreConfig, LOCAL_FS_STORE_CONFIG_FIELDS) + +/** + * @param rootDir Fallback if not in `params` + */ +static LocalFSStoreConfigT localFSStoreConfigDefaults( + const Path & storeDir, + const std::optional & rootDir) +{ + return { + .rootDir = {std::nullopt}, + .stateDir = {rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir}, + .logDir = {rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir}, + .realStoreDir = {rootDir ? *rootDir + "/nix/store" : storeDir}, + }; +} + +static LocalFSStoreConfigT localFSStoreConfigApplyParse( + const Path & storeDir, + LocalFSStoreConfigT parsed) +{ + auto defaults = localFSStoreConfigDefaults( + storeDir, + parsed.rootDir.value_or(std::nullopt)); + return {LOCAL_FS_STORE_CONFIG_FIELDS(APPLY_ROW)}; +} + +config::SettingDescriptionMap LocalFSStoreConfig::descriptions() +{ + constexpr auto & descriptions = localFSStoreConfigDescriptions; + auto defaults = localFSStoreConfigDefaults(settings.nixStore, std::nullopt); + return { + LOCAL_FS_STORE_CONFIG_FIELDS(DESCRIBE_ROW) + }; +} + +LocalFSStore::Config::LocalFSStoreConfig( + const Store::Config & storeConfig, + const StoreReference::Params & params) + : LocalFSStoreConfigT{ + localFSStoreConfigApplyParse( + storeConfig.storeDir, + localFSStoreConfigParse(params))} + , storeConfig{storeConfig} +{ +} + +static LocalFSStoreConfigT applyAuthority( + LocalFSStoreConfigT parsed, + PathView rootDir) +{ + if (!rootDir.empty()) + parsed.rootDir = std::optional{Path{rootDir}}; + return parsed; +} + +LocalFSStore::Config::LocalFSStoreConfig( + const Store::Config & storeConfig, + PathView rootDir, + const StoreReference::Params & params) + : LocalFSStoreConfigT{ + localFSStoreConfigApplyParse( + storeConfig.storeDir, + applyAuthority( + localFSStoreConfigParse(params), + rootDir))} + , storeConfig{storeConfig} { } diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index e40c5fa6e6a..29ea880dc4d 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -5,11 +5,106 @@ #include "nix/store/realisation.hh" #include "nix/util/processes.hh" #include "nix/util/url.hh" -#include "nix/store/store-open.hh" +#include "nix/store/store-api.hh" #include "nix/store/store-registration.hh" +#include "nix/store/config-parse-impl.hh" namespace nix { +static LocalOverlayStoreConfigT localOverlayStoreConfigDescriptions = { + .lowerStoreConfig{ + .name = "lower-store", + .description = R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). + + Must be a store with a store dir on the file system. + Must be used as OverlayFS lower layer for this store's store dir. + )", + // It's not actually machine-specific, but we don't yet have a + // `to_json` for `StoreConfig`. + .documentDefault = false, + }, + .upperLayer{ + .name = "upper-layer", + .description = R"( + Directory containing the OverlayFS upper layer for this store's store dir. + )", + }, + .checkMount{ + .name = "check-mount", + .description = R"( + Check that the overlay filesystem is correctly mounted. + + Nix does not manage the overlayfs mount point itself, but the correct + functioning of the overlay store does depend on this mount point being set up + correctly. Rather than just assume this is the case, check that the lowerdir + and upperdir options are what we expect them to be. This check is on by + default, but can be disabled if needed. + )", + }, + .remountHook{ + .name = "remount-hook", + .description = R"( + Script or other executable to run when overlay filesystem needs remounting. + + This is occasionally necessary when deleting a store path that exists in both upper and lower layers. + In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly + is the only way to perform the deletion without creating a "whiteout". + However this causes the OverlayFS kernel data structures to get out-of-sync, + and can lead to 'stale file handle' errors; remounting solves the problem. + + The store directory is passed as an argument to the invoked executable. + )", + }, +}; + +#define LOCAL_OVERLAY_STORE_CONFIG_FIELDS(X) \ + X(lowerStoreConfig), \ + X(upperLayer), \ + X(checkMount), \ + X(remountHook), + +MAKE_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS) + +static LocalOverlayStoreConfigT localOverlayStoreConfigDefaults() +{ + return { + .lowerStoreConfig = {make_ref(StoreReference::Params{})}, + .upperLayer = {""}, + .checkMount = {true}, + .remountHook = {""}, + }; +} + +MAKE_APPLY_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap LocalOverlayStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + ret.merge(LocalStoreConfig::descriptions()); + { + constexpr auto & descriptions = localOverlayStoreConfigDescriptions; + auto defaults = localOverlayStoreConfigDefaults(); + ret.merge(decltype(ret){ + LOCAL_OVERLAY_STORE_CONFIG_FIELDS(DESCRIBE_ROW) + }); + } + return ret; +} + +LocalOverlayStore::Config::LocalOverlayStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreReference::Params & params) + : LocalStore::Config(scheme, authority, params) + , LocalOverlayStoreConfigT{localOverlayStoreConfigApplyParse(params)} +{ +} + + std::string LocalOverlayStoreConfig::doc() { return @@ -36,7 +131,7 @@ LocalOverlayStore::LocalOverlayStore(ref config) , LocalFSStore{*config} , LocalStore{static_cast>(config)} , config{config} - , lowerStore(openStore(percentDecode(config->lowerStoreUri.get())).dynamic_pointer_cast()) + , lowerStore(config->lowerStoreConfig.value->openStore().dynamic_pointer_cast()) { if (config->checkMount.get()) { std::smatch match; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 76fadba8649..bf9d782def6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -17,6 +17,7 @@ #include "nix/util/posix-source-accessor.hh" #include "nix/store/keys.hh" #include "nix/util/users.hh" +#include "nix/store/config-parse-impl.hh" #include "nix/store/store-open.hh" #include "nix/store/store-registration.hh" @@ -61,15 +62,72 @@ namespace nix { -LocalStoreConfig::LocalStoreConfig( +constexpr static const LocalStoreConfigT localStoreConfigDescriptions = { + .requireSigs = { + .name = "require-sigs", + .description = "Whether store paths copied into this store should have a trusted signature.", + }, + .readOnly = { + .name = "read-only", + .description = R"( + Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. + + Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. + + Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. + + > **Warning** + > Do not use this unless the filesystem is read-only. + > + > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. + > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. + )", + }, +}; + +#define LOCAL_STORE_CONFIG_FIELDS(X) \ + X(requireSigs), \ + X(readOnly), + +MAKE_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS) + +static LocalStoreConfigT localStoreConfigDefaults() +{ + return { + .requireSigs = {settings.requireSigs}, + .readOnly = {false}, + }; +} + +MAKE_APPLY_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap LocalStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + { + constexpr auto & descriptions = localStoreConfigDescriptions; + auto defaults = localStoreConfigDefaults(); + ret.merge(decltype(ret){ + LOCAL_STORE_CONFIG_FIELDS(DESCRIBE_ROW) + }); + } + return ret; +} + +LocalStore::Config::LocalStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(authority, params) + const StoreReference::Params & params) + : Store::Config(params) + , LocalFSStore::Config(*this, authority, params) + , LocalStoreConfigT{localStoreConfigApplyParse(params)} { } +LocalStoreConfig::LocalStoreConfig(const LocalStoreConfig &) = default; + std::string LocalStoreConfig::doc() { return @@ -916,7 +974,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) for (auto & sub : getDefaultSubstituters()) { if (remaining.empty()) break; if (sub->storeDir != storeDir) continue; - if (!sub->config.wantMassQuery) continue; + if (!sub->resolvedSubstConfig.wantMassQuery) continue; auto valid = sub->queryValidPaths(remaining); diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 483b337bf21..ec8a1daaf34 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -71,8 +71,8 @@ StoreReference Machine::completeStoreReference() const auto * generic = std::get_if(&storeUri.variant); if (generic && generic->scheme == "ssh") { - storeUri.params["max-connections"] = "1"; - storeUri.params["log-fd"] = "4"; + storeUri.params["max-connections"] = 1; + storeUri.params["log-fd"] = 4; } if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) { @@ -84,14 +84,10 @@ StoreReference Machine::completeStoreReference() const { auto & fs = storeUri.params["system-features"]; - auto append = [&](auto feats) { - for (auto & f : feats) { - if (fs.size() > 0) fs += ' '; - fs += f; - } - }; - append(supportedFeatures); - append(mandatoryFeatures); + if (!fs.is_array()) fs = nlohmann::json::array(); + auto features = supportedFeatures; + features.insert(supportedFeatures.begin(), supportedFeatures.end()); + for (auto & feat : features) fs += feat; } return storeUri; diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 94b8951fdd9..4f99a0c670e 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -265,6 +265,7 @@ sources = files( 'builtins/unpack-channel.cc', 'common-protocol.cc', 'common-ssh-store-config.cc', + 'config-parse.cc', 'content-address.cc', 'daemon.cc', 'derivations.cc', diff --git a/src/libstore/mounted-ssh-store.md b/src/libstore/mounted-ssh-store.md deleted file mode 100644 index 1ebfe3081dc..00000000000 --- a/src/libstore/mounted-ssh-store.md +++ /dev/null @@ -1,18 +0,0 @@ -R"( - -**Store URL format**: `mounted-ssh-ng://[username@]hostname` - -Experimental store type that allows full access to a Nix store on a remote machine, -and additionally requires that store be mounted in the local file system. - -The mounting of that store is not managed by Nix, and must by managed manually. -It could be accomplished with SSHFS or NFS, for example. - -The local file system is used to optimize certain operations. -For example, rather than serializing Nix archives and sending over the Nix channel, -we can directly access the file system data via the mount-point. - -The local file system is also used to make certain operations possible that wouldn't otherwise be. -For example, persistent GC roots can be created if they reside on the same file system as the remote store: -the remote side will create the symlinks necessary to avoid race conditions. -)" diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 3151f319c00..8af716a6f77 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -18,14 +18,64 @@ #include "nix/util/callback.hh" #include "nix/store/filetransfer.hh" #include "nix/util/signals.hh" +#include "nix/store/config-parse-impl.hh" #include namespace nix { +constexpr static const RemoteStoreConfigT remoteStoreConfigDescriptions = { + .maxConnections{ + .name = "max-connections", + .description = "Maximum number of concurrent connections to the Nix daemon.", + }, + .maxConnectionAge{ + .name = "max-connection-age", + .description = "Maximum age of a connection before it is closed.", + }, +}; + + +#define REMOTE_STORE_CONFIG_FIELDS(X) \ + X(maxConnections), \ + X(maxConnectionAge), + + +MAKE_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS) + + +static RemoteStoreConfigT remoteStoreConfigDefaults() +{ + return { + .maxConnections = {1}, + .maxConnectionAge = {std::numeric_limits::max()}, + }; +} + + +MAKE_APPLY_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS) + + +config::SettingDescriptionMap RemoteStoreConfig::descriptions() +{ + constexpr auto & descriptions = remoteStoreConfigDescriptions; + auto defaults = remoteStoreConfigDefaults(); + return { + REMOTE_STORE_CONFIG_FIELDS(DESCRIBE_ROW) + }; +} + + +RemoteStore::Config::RemoteStoreConfig(const Store::Config & storeConfig, const StoreReference::Params & params) + : RemoteStoreConfigT{remoteStoreConfigApplyParse(params)} + , storeConfig{storeConfig} +{ +} + + /* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Config & config) - : Store{config} + : Store{config.storeConfig} , config{config} , connections(make_ref>( std::max(1, config.maxConnections.get()), diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 618112d1c07..27263f6f12d 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -2,8 +2,6 @@ #if NIX_WITH_S3_SUPPORT -#include - #include "nix/store/s3.hh" #include "nix/store/nar-info.hh" #include "nix/store/nar-info-disk-cache.hh" @@ -11,6 +9,7 @@ #include "nix/util/compression.hh" #include "nix/store/filetransfer.hh" #include "nix/util/signals.hh" +#include "nix/store/config-parse-impl.hh" #include "nix/store/store-registration.hh" #include @@ -237,25 +236,133 @@ S3Helper::FileTransferResult S3Helper::getObject( } -S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( - std::string_view uriScheme, - std::string_view bucketName, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , bucketName(bucketName) +constexpr static const S3BinaryCacheStoreConfigT s3BinaryCacheStoreConfigDescriptions = { + .profile{ + .name = "profile", + .description = R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )", + }, + .region{ + .name = "region", + .description = R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )", + }, + .scheme{ + .name = "scheme", + .description = R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )", + }, + .endpoint{ + .name = "endpoint", + .description = R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )", + }, + .narinfoCompression{ + .name = "narinfo-compression", + .description = "Compression method for `.narinfo` files.", + }, + .lsCompression{ + .name = "ls-compression", + .description = "Compression method for `.ls` files.", + }, + .logCompression{ + .name = "log-compression", + .description = R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )", + }, + .multipartUpload{ + .name = "multipart-upload", + .description = "Whether to use multi-part uploads.", + }, + .bufferSize{ + .name = "buffer-size", + .description = "Size (in bytes) of each part in multi-part uploads.", + }, +}; + +#define S3_BINARY_CACHE_STORE_CONFIG_FIELDS(X) \ + X(profile), \ + X(region), \ + X(scheme), \ + X(endpoint), \ + X(narinfoCompression), \ + X(lsCompression), \ + X(logCompression), \ + X(multipartUpload), \ + X(bufferSize), + +MAKE_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS) + +static S3BinaryCacheStoreConfigT s3BinaryCacheStoreConfigDefaults() +{ + return { + .profile = {""}, + .region = {Aws::Region::US_EAST_1}, + .scheme = {""}, + .endpoint = {""}, + .narinfoCompression = {""}, + .lsCompression = {""}, + .logCompression = {""}, + .multipartUpload = {false}, + .bufferSize = {5 * 1024 * 1024}, + }; +} + +MAKE_APPLY_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap S3BinaryCacheStoreConfig::descriptions() { - // Don't want to use use AWS SDK in header, so we check the default - // here. TODO do this better after we overhaul the store settings - // system. - assert(std::string{defaultRegion} == std::string{Aws::Region::US_EAST_1}); + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(BinaryCacheStoreConfig::descriptions()); + { + constexpr auto & descriptions = s3BinaryCacheStoreConfigDescriptions; + auto defaults = s3BinaryCacheStoreConfigDefaults(); + ret.merge(decltype(ret){ + S3_BINARY_CACHE_STORE_CONFIG_FIELDS(DESCRIBE_ROW) + }); + } + return ret; +} +S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreReference::Params & params) + : Store::Config{params} + , BinaryCacheStore::Config{*this, params} + , S3BinaryCacheStoreConfigT{s3BinaryCacheStoreConfigApplyParse(params)} + , bucketName{authority} +{ if (bucketName.empty()) - throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); + throw UsageError("`%s` store requires a bucket name in its Store URI", scheme); } -S3BinaryCacheStore::S3BinaryCacheStore(ref config) +S3BinaryCacheStore::S3BinaryCacheStore(ref config) : BinaryCacheStore(*config) , config{config} { } @@ -274,7 +381,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore S3Helper s3Helper; - S3BinaryCacheStoreImpl(ref config) + S3BinaryCacheStoreImpl(ref config) : Store{*config} , BinaryCacheStore{*config} , S3BinaryCacheStore{config} @@ -293,12 +400,14 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore void init() override { if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) { - config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); - config->priority.setDefault(cacheInfo->priority); + resolvedSubstConfig.wantMassQuery.value = + config->storeConfig.wantMassQuery.value_or(cacheInfo->wantMassQuery); + resolvedSubstConfig.priority.value = + config->storeConfig.priority.value_or(cacheInfo->priority); } else { BinaryCacheStore::init(); diskCache->createCache( - getUri(), config->storeDir, config->wantMassQuery, config->priority); + getUri(), config->storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority); } } @@ -585,10 +694,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore ref S3BinaryCacheStoreImpl::Config::openStore() const { - return make_ref(ref{ - // FIXME we shouldn't actually need a mutable config - std::const_pointer_cast(shared_from_this()) - }); + return make_ref(ref{shared_from_this()}); } static RegisterStoreImplementation regS3BinaryCacheStore; diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 753256d483e..494c5719185 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,3 +1,4 @@ +#include "nix/util/json-utils.hh" #include "nix/store/ssh-store.hh" #include "nix/store/local-fs-store.hh" #include "nix/store/remote-store-connection.hh" @@ -7,17 +8,101 @@ #include "nix/store/worker-protocol-impl.hh" #include "nix/util/pool.hh" #include "nix/store/ssh.hh" +#include "nix/store/config-parse-impl.hh" #include "nix/store/store-registration.hh" namespace nix { +constexpr static const SSHStoreConfigT sshStoreConfigDescriptions = { + .remoteProgram{ + .name = "remote-program", + .description = "Path to the `nix-daemon` executable on the remote machine.", + }, +}; + + +#define SSH_STORE_CONFIG_FIELDS(X) \ + X(remoteProgram) + + +MAKE_PARSE(SSHStoreConfig, sshStoreConfig, SSH_STORE_CONFIG_FIELDS) + + +static SSHStoreConfigT sshStoreConfigDefaults() +{ + return { + .remoteProgram = {{"nix-daemon"}}, + }; +} + + +MAKE_APPLY_PARSE(SSHStoreConfig, sshStoreConfig, SSH_STORE_CONFIG_FIELDS) + + +config::SettingDescriptionMap SSHStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(CommonSSHStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + { + constexpr auto & descriptions = sshStoreConfigDescriptions; + auto defaults = sshStoreConfigDefaults(); + ret.merge(decltype(ret){ + SSH_STORE_CONFIG_FIELDS(DESCRIBE_ROW) + }); + } + ret.insert_or_assign( + "mounted", + config::SettingDescription{ + .description = stripIndentation(R"( + If this nested settings object is defined (`{..}` not `null`), additionally requires that store be mounted in the local file system. + + The mounting of that store is not managed by Nix, and must by managed manually. + It could be accomplished with SSHFS or NFS, for example. + + The local file system is used to optimize certain operations. + For example, rather than serializing Nix archives and sending over the Nix channel, + we can directly access the file system data via the mount-point. + + The local file system is also used to make certain operations possible that wouldn't otherwise be. + For example, persistent GC roots can be created if they reside on the same file system as the remote store: + the remote side will create the symlinks necessary to avoid race conditions. + )"), + .experimentalFeature = Xp::MountedSSHStore, + .info = config::SettingDescription::Sub{ + .nullable = true, + .map = LocalFSStoreConfig::descriptions() + }, + }); + return ret; +} + + +static std::optional getMounted( + const Store::Config & storeConfig, + const StoreReference::Params & params, + const ExperimentalFeatureSettings & xpSettings) +{ + auto mountedParamsOpt = optionalValueAt(params, "mounted"); + if (!mountedParamsOpt) return {}; + auto * mountedParamsP = getNullable(*mountedParamsOpt); + xpSettings.require(Xp::MountedSSHStore); + if (!mountedParamsP) return {}; + auto & mountedParams = getObject(*mountedParamsP); + return {{storeConfig, mountedParams}}; +} + + SSHStoreConfig::SSHStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params) + const StoreReference::Params & params, const ExperimentalFeatureSettings & xpSettings) : Store::Config{params} - , RemoteStore::Config{params} + , RemoteStore::Config{*this, params} , CommonSSHStoreConfig{scheme, authority, params} + , SSHStoreConfigT{sshStoreConfigApplyParse(params)} + , mounted{getMounted(*this, params, xpSettings)} { } @@ -87,32 +172,6 @@ struct SSHStore : virtual RemoteStore }; -MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) -{ -} - -MountedSSHStoreConfig::MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(scheme, host, params) - , LocalFSStoreConfig(params) -{ -} - -std::string MountedSSHStoreConfig::doc() -{ - return - #include "mounted-ssh-store.md" - ; -} - - /** * The mounted ssh store assumes that filesystems on the remote host are * shared with the local host. This means that the remote nix store is @@ -129,13 +188,16 @@ std::string MountedSSHStoreConfig::doc() */ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore { - using Config = MountedSSHStoreConfig; + using Config = SSHStore::Config; + + const LocalFSStore::Config & mountedConfig; - MountedSSHStore(ref config) + MountedSSHStore(ref config, const LocalFSStore::Config & mountedConfig) : Store{*config} , RemoteStore{*config} , SSHStore{config} - , LocalFSStore{*config} + , LocalFSStore{mountedConfig} + , mountedConfig{mountedConfig} { extraRemoteProgramArgs = { "--process-ops", @@ -184,14 +246,13 @@ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore }; -ref SSHStore::Config::openStore() const { - return make_ref(ref{shared_from_this()}); -} - ref MountedSSHStore::Config::openStore() const { - return make_ref(ref{ - std::dynamic_pointer_cast(shared_from_this()) - }); + ref config {shared_from_this()}; + + if (config->mounted) + return make_ref(config, *config->mounted); + else + return make_ref(config); } @@ -213,6 +274,5 @@ ref SSHStore::openConnection() } static RegisterStoreImplementation regSSHStore; -static RegisterStoreImplementation regMountedSSHStore; } diff --git a/src/libstore/ssh-store.md b/src/libstore/ssh-store.md index 881537e7114..ea34ef2b710 100644 --- a/src/libstore/ssh-store.md +++ b/src/libstore/ssh-store.md @@ -4,5 +4,4 @@ R"( Experimental store type that allows full access to a Nix store on a remote machine. - )" diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b1360f8e591..d4042f8de48 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -19,6 +19,7 @@ #include "nix/store/worker-protocol.hh" #include "nix/util/signals.hh" #include "nix/util/users.hh" +#include "nix/store/config-parse-impl.hh" #include #include @@ -29,7 +30,6 @@ using json = nlohmann::json; namespace nix { - bool MixStoreDirMethods::isInStore(PathView path) const { return isInDir(path, storeDir); @@ -189,6 +189,114 @@ std::pair MixStoreDirMethods::computeStorePath( } +constexpr static const StoreConfigT storeConfigDescriptions = { + .pathInfoCacheSize{ + .name = "path-info-cache-size", + .description = "Size of the in-memory store path metadata cache.", + }, + .isTrusted{ + .name = "trusted", + .description = R"( + Whether paths from this store can be used as substitutes + even if they are not signed by a key listed in the + [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) + setting. + )", + }, + .systemFeatures{ + .name = "system-features", + .description = R"( + Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. + + Example: `"kvm"` + )", + // The default value is CPU- and OS-specific, and thus + // unsuitable to be rendered in the documentation. + .documentDefault = false, + }, +}; + +constexpr static const SubstituterConfigT substituterConfigDescriptions = { + .priority{ + .name = "priority", + .description = R"( + Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + A lower value means a higher priority. + )", + }, + .wantMassQuery{ + .name = "want-mass-query", + .description = R"( + Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + )", + }, +}; + + +#define STORE_CONFIG_FIELDS(X) \ + X(pathInfoCacheSize), \ + X(isTrusted), \ + X(systemFeatures), + +#define SUBSTITUTER_CONFIG_FIELDS(X) \ + X(priority), \ + X(wantMassQuery), + + +MAKE_PARSE(StoreConfig, storeConfig, STORE_CONFIG_FIELDS) +MAKE_PARSE(SubstituterConfig, substituterConfig, SUBSTITUTER_CONFIG_FIELDS) + + +static StoreConfigT storeConfigDefaults() +{ + return { + .pathInfoCacheSize = {65536}, + .isTrusted = {false}, + .systemFeatures = {StoreConfig::getDefaultSystemFeatures()}, + }; +}; + +SubstituterConfigT substituterConfigDefaults() +{ + return { + .priority = {0}, + .wantMassQuery = {false}, + }; +}; + + +MAKE_APPLY_PARSE(StoreConfig, storeConfig, STORE_CONFIG_FIELDS) + + +Store::Config::StoreConfig(const StoreReference::Params & params) + : StoreDirConfig{params} + , StoreConfigT{storeConfigApplyParse(params)} + , SubstituterConfigT{substituterConfigParse(params)} +{ +} + + +config::SettingDescriptionMap StoreConfig::descriptions() +{ + auto ret = StoreDirConfig::descriptions(); + { + constexpr auto & descriptions = storeConfigDescriptions; + auto defaults = storeConfigDefaults(); + ret.merge(config::SettingDescriptionMap { + STORE_CONFIG_FIELDS(DESCRIBE_ROW) + }); + } + { + constexpr auto & descriptions = substituterConfigDescriptions; + auto defaults = substituterConfigDefaults(); + ret.merge(config::SettingDescriptionMap { + SUBSTITUTER_CONFIG_FIELDS(DESCRIBE_ROW) + }); + }; + return ret; +} + + StorePath Store::addToStore( std::string_view name, const SourcePath & path, diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index ec65013ef2a..9ef8dc32d39 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -1,13 +1,45 @@ #include "nix/store/store-dir-config.hh" +#include "nix/store/config-parse-impl.hh" #include "nix/util/util.hh" #include "nix/store/globals.hh" namespace nix { -StoreDirConfig::StoreDirConfig(const Params & params) - : StoreDirConfigBase(params) - , MixStoreDirMethods{storeDir_} +constexpr static const StoreDirConfigT storeDirConfigDescriptions = { + ._storeDir{ + .name = "store", + .description = R"( + Logical location of the Nix store, usually + `/nix/store`. Note that you can only copy store paths + between stores if they have the same `store` setting. + )", + }, +}; + +#define STORE_DIR_CONFIG_FIELDS(X) X(_storeDir), + +MAKE_PARSE(StoreDirConfig, storeDirConfig, STORE_DIR_CONFIG_FIELDS) + +static StoreDirConfigT storeDirConfigDefaults() +{ + return { + ._storeDir = {settings.nixStore}, + }; +} + +MAKE_APPLY_PARSE(StoreDirConfig, storeDirConfig, STORE_DIR_CONFIG_FIELDS) + +StoreDirConfig::StoreDirConfig(const StoreReference::Params & params) + : StoreDirConfigT{storeDirConfigApplyParse(params)} + , MixStoreDirMethods{_storeDir.value} +{ +} + +config::SettingDescriptionMap StoreDirConfig::descriptions() { + constexpr auto & descriptions = storeDirConfigDescriptions; + auto defaults = storeDirConfigDefaults(); + return {STORE_DIR_CONFIG_FIELDS(DESCRIBE_ROW)}; } } diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index cb4e2cfb8eb..eaa972c7d44 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -1,13 +1,18 @@ #include +#include + #include "nix/util/error.hh" #include "nix/util/url.hh" #include "nix/store/store-reference.hh" #include "nix/util/file-system.hh" #include "nix/util/util.hh" +#include "nix/util/json-utils.hh" namespace nix { +bool StoreReference::operator==(const StoreReference & rhs) const = default; + static bool isNonUriPath(const std::string & spec) { return @@ -33,20 +38,96 @@ std::string StoreReference::render() const }, variant); + StringMap params2; + for (auto & [k, v] : params) { + auto * p = v.get_ptr(); + // if it is a JSON string, just use that + + // FIXME: Ensure the literal string isn't itself valid JSON. If + // it is, we still need to dump to escape it. + params2.insert_or_assign(k, p ? *p : v.dump()); + } + if (!params.empty()) { res += "?"; - res += encodeQuery(params); + res += encodeQuery(params2); } return res; } +static StoreReference::Params decodeParamsJson(StringMap paramsRaw) +{ + StoreReference::Params params; + for (auto && [k, v] : std::move(paramsRaw)) { + nlohmann::json j; + /* We have to parse the URL un an "untyped" way before we do a + "typed" conversion to specific store-configuration types. As + such, the best we can do for back-compat is just white-list + specific query parameter names. + + These are all the boolean store parameters in use at the time + of the introduction of JSON store configuration, as evidenced + by `git grep 'F'`. So these will continue working with + "yes"/"no"/"1"/"0", whereas any new ones will require + "true"/"false". + */ + bool preJsonBool = + std::set{ + "check-mount", + "compress", + "trusted", + "multipart-upload", + "parallel-compression", + "read-only", + "require-sigs", + "want-mass-query", + "index-debug-info", + "write-nar-listing", + } + .contains(std::string_view{k}); + + auto warnPreJson = [&] { + warn( + "in query param '%s', using '%s' to mean a boolean is deprecated, please use valid JSON 'true' or 'false'", + k, + v); + }; + + if (preJsonBool && (v == "yes" || v == "1")) { + j = true; + warnPreJson(); + } else if (preJsonBool && (v == "no" || v == "0")) { + j = true; + warnPreJson(); + } else { + try { + j = nlohmann::json::parse(v); + } catch (nlohmann::json::exception &) { + // if its not valid JSON... + if (k == "remote-program" || k == "system-features") { + // Back compat hack! Split and take that array + j = tokenizeString>(v); + } else { + // ...keep the literal string. + j = std::move(v); + } + } + } + params.insert_or_assign(std::move(k), std::move(j)); + } + return params; +} + StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams) { auto params = extraParams; try { auto parsedUri = parseURL(uri); - params.insert(parsedUri.query.begin(), parsedUri.query.end()); + { + auto params2 = decodeParamsJson(std::move(parsedUri.query)); + params.insert(params2.begin(), params2.end()); + } auto baseURI = parsedUri.authority.value_or("") + parsedUri.path; @@ -104,13 +185,75 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen std::pair splitUriAndParams(const std::string & uri_) { auto uri(uri_); - StoreReference::Params params; + StringMap params; auto q = uri.find('?'); if (q != std::string::npos) { params = decodeQuery(uri.substr(q + 1)); uri = uri_.substr(0, q); } - return {uri, params}; + return {uri, decodeParamsJson(std::move(params))}; +} + +} + +namespace nlohmann { + +StoreReference adl_serializer::from_json(const json & json) +{ + StoreReference ref; + switch (json.type()) { + + case json::value_t::string: { + ref = StoreReference::parse(json.get_ref()); + break; + } + + case json::value_t::object: { + auto & obj = getObject(json); + auto scheme = getString(valueAt(obj, "scheme")); + auto variant = scheme == "auto" ? (StoreReference::Variant{StoreReference::Auto{}}) + : (StoreReference::Variant{StoreReference::Specified{ + .scheme = scheme, + .authority = getString(valueAt(obj, "authority")), + }}); + auto params = obj; + params.erase("scheme"); + params.erase("authority"); + ref = StoreReference{ + .variant = std::move(variant), + .params = std::move(params), + }; + break; + } + + case json::value_t::null: + case json::value_t::number_unsigned: + case json::value_t::number_integer: + case json::value_t::number_float: + case json::value_t::boolean: + case json::value_t::array: + case json::value_t::binary: + case json::value_t::discarded: + default: + throw UsageError( + "Invalid JSON for Store configuration: is type '%s' but must be string or object", json.type_name()); + }; + + return ref; +} + +void adl_serializer::to_json(json & obj, StoreReference s) +{ + obj = s.params; + std::visit( + overloaded{ + [&](const StoreReference::Auto &) { obj.emplace("scheme", "auto"); }, + [&](const StoreReference::Specified & g) { + obj.emplace("scheme", g.scheme); + obj.emplace("authority", g.authority); + }, + }, + s.variant); } } diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc index 6362ac0365b..cc2fa848495 100644 --- a/src/libstore/store-registration.cc +++ b/src/libstore/store-registration.cc @@ -2,19 +2,18 @@ #include "nix/store/store-open.hh" #include "nix/store/local-store.hh" #include "nix/store/uds-remote-store.hh" +#include "nix/util/json-utils.hh" namespace nix { -ref openStore(const std::string & uri, const Store::Config::Params & extraParams) +ref openStore(const std::string & uri, const StoreReference::Params & extraParams) { return openStore(StoreReference::parse(uri, extraParams)); } ref openStore(StoreReference && storeURI) { - auto store = resolveStoreConfig(std::move(storeURI))->openStore(); - store->init(); - return store; + return resolveStoreConfig(std::move(storeURI))->openStore(); } ref resolveStoreConfig(StoreReference && storeURI) @@ -24,7 +23,7 @@ ref resolveStoreConfig(StoreReference && storeURI) auto storeConfig = std::visit( overloaded{ [&](const StoreReference::Auto &) -> ref { - auto stateDir = getOr(params, "state", settings.nixStateDir); + auto stateDir = getString(getOr(params, "state", settings.nixStateDir)); if (access(stateDir.c_str(), R_OK | W_OK) == 0) return make_ref(params); else if (pathExists(settings.nixDaemonSocketFile)) @@ -53,7 +52,7 @@ ref resolveStoreConfig(StoreReference && storeURI) return make_ref(params); }, [&](const StoreReference::Specified & g) { - for (const auto & [storeName, implem] : Implementations::registered()) + for (auto & [name, implem] : Implementations::registered()) if (implem.uriSchemes.count(g.scheme)) return implem.parseConfig(g.scheme, g.authority, params); @@ -63,7 +62,6 @@ ref resolveStoreConfig(StoreReference && storeURI) storeURI.variant); experimentalFeatureSettings.require(storeConfig->experimentalFeature()); - storeConfig->warnUnknownSettings(); return storeConfig; } @@ -85,10 +83,12 @@ std::list> getDefaultSubstituters() } }; - for (const auto & uri : settings.substituters.get()) + for (auto & uri : settings.substituters.get()) addStore(uri); - stores.sort([](ref & a, ref & b) { return a->config.priority < b->config.priority; }); + stores.sort([](ref & a, ref & b) { + return a->resolvedSubstConfig.priority < b->resolvedSubstConfig.priority; + }); return stores; }()); @@ -103,3 +103,20 @@ Implementations::Map & Implementations::registered() } } + +namespace nlohmann { + +using namespace nix::config; + +ref adl_serializer>::from_json(const json & json) +{ + return resolveStoreConfig(adl_serializer::from_json(json)); +} + +void adl_serializer>::to_json(json & obj, ref s) +{ + // TODO, for tests maybe + assert(false); +} + +} diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 9950e38b36c..158232b0f98 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -18,16 +18,26 @@ namespace nix { +config::SettingDescriptionMap UDSRemoteStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + return ret; +} + + UDSRemoteStoreConfig::UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, const StoreReference::Params & params) : Store::Config{params} - , LocalFSStore::Config{params} - , RemoteStore::Config{params} + , LocalFSStore::Config{*this, params} + , RemoteStore::Config{*this, params} , path{authority.empty() ? settings.nixDaemonSocketFile : authority} { - if (uriSchemes().count(scheme) == 0) { + if (uriSchemes().count(std::string{scheme}) == 0) { throw UsageError("Scheme must be 'unix'"); } } @@ -41,16 +51,6 @@ std::string UDSRemoteStoreConfig::doc() } -// A bit gross that we now pass empty string but this is knowing that -// empty string will later default to the same nixDaemonSocketFile. Why -// don't we just wire it all through? I believe there are cases where it -// will live reload so we want to continue to account for that. -UDSRemoteStoreConfig::UDSRemoteStoreConfig(const Params & params) - : UDSRemoteStoreConfig(*uriSchemes().begin(), "", params) -{ -} - - UDSRemoteStore::UDSRemoteStore(ref config) : Store{*config} , LocalFSStore{*config} diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 81e0a2584f6..860682aa4eb 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1600,9 +1600,9 @@ void DerivationBuilderImpl::startDaemon() auto store = makeRestrictedStore( [&]{ auto config = make_ref(*getLocalStore().config); - config->pathInfoCacheSize = 0; - config->stateDir = "/no-such-path"; - config->logDir = "/no-such-path"; + config->pathInfoCacheSize.value = 0; + config->stateDir.value = "/no-such-path"; + config->logDir.value = "/no-such-path"; return config; }(), ref(std::dynamic_pointer_cast(this->store.shared_from_this())), diff --git a/src/libutil/include/nix/util/config-abstract.hh b/src/libutil/include/nix/util/config-abstract.hh new file mode 100644 index 00000000000..ba6bee740f7 --- /dev/null +++ b/src/libutil/include/nix/util/config-abstract.hh @@ -0,0 +1,45 @@ +#pragma once +///@type + +#include + +namespace nix::config { + +template +struct PlainValue +{ + T value; + + operator const T &() const & + { + return value; + } + + operator T &() & + { + return value; + } + + const T & get() const + { + return value; + } + + bool operator==(auto && v2) const + { + return value == v2; + } + + bool operator!=(auto && v2) const + { + return value != v2; + } +}; + +template +auto && operator<<(auto && str, const PlainValue & opt) +{ + return str << opt.get(); +} + +} diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libutil/include/nix/util/configuration.hh index 24b42f02c84..dc939dfcea1 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libutil/include/nix/util/configuration.hh @@ -180,13 +180,12 @@ public: const std::string name; const std::string description; const StringSet aliases; + std::optional experimentalFeature; int created = 123; bool overridden = false; - std::optional experimentalFeature; - protected: AbstractSetting( diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index e30b8dacd48..d1a683e4470 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -16,6 +16,7 @@ headers = files( 'comparator.hh', 'compression.hh', 'compute-levels.hh', + 'config-abstract.hh', 'config-global.hh', 'config-impl.hh', 'configuration.hh', diff --git a/src/libutil/include/nix/util/util.hh b/src/libutil/include/nix/util/util.hh index 2361bf2e773..37d14d4fb2b 100644 --- a/src/libutil/include/nix/util/util.hh +++ b/src/libutil/include/nix/util/util.hh @@ -223,18 +223,18 @@ std::pair getLine(std::string_view s); /** * Get a value for the specified key from an associate container. */ -template -const typename T::mapped_type * get(const T & map, const typename T::key_type & key) +template +const typename T::mapped_type * get(const T & map, K && key) { - auto i = map.find(key); + auto i = map.find(std::forward(key)); if (i == map.end()) return nullptr; return &i->second; } -template -typename T::mapped_type * get(T & map, const typename T::key_type & key) +template +typename T::mapped_type * get(T & map, K && key) { - auto i = map.find(key); + auto i = map.find(std::forward(key)); if (i == map.end()) return nullptr; return &i->second; } @@ -242,12 +242,12 @@ typename T::mapped_type * get(T & map, const typename T::key_type & key) /** * Get a value for the specified key from an associate container, or a default value if the key isn't present. */ -template +template const typename T::mapped_type & getOr(T & map, - const typename T::key_type & key, + K && key, const typename T::mapped_type & defaultValue) { - auto i = map.find(key); + auto i = map.find(std::forward(key)); if (i == map.end()) return defaultValue; return i->second; } diff --git a/src/nix/main.cc b/src/nix/main.cc index 05c5da27d6b..551f1918a04 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -198,7 +198,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs auto & j = stores[storeName]; j["doc"] = implem.doc; j["uri-schemes"] = implem.uriSchemes; - j["settings"] = implem.getConfig()->toJSON(); + j["settings"] = implem.configDescriptions(); j["experimentalFeature"] = implem.experimentalFeature; } res["stores"] = std::move(stores); diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 301f8aa50ca..98f30230090 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -245,9 +245,9 @@ static PeerInfo getPeerInfo(int remote) */ static ref openUncachedStore() { - Store::Config::Params params; // FIXME: get params from somewhere + StoreReference::Params params; // FIXME: get params from somewhere // Disable caching since the client already does that. - params["path-info-cache-size"] = "0"; + params["path-info-cache-size"] = 0; return openStore(settings.storeUri, params); } diff --git a/tests/functional/build-remote-with-mounted-ssh-ng.sh b/tests/functional/build-remote-with-mounted-ssh-ng.sh index e2627af394c..9f25f7816b3 100755 --- a/tests/functional/build-remote-with-mounted-ssh-ng.sh +++ b/tests/functional/build-remote-with-mounted-ssh-ng.sh @@ -5,17 +5,25 @@ source common.sh requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" +# An example of a command that uses the store only for its settings, to +# make sure we catch needing the XP feature early. +touch "$TEST_ROOT/foo" +expectStderr 1 nix --store 'ssh-ng://localhost?mounted=%7B%7D' store add "$TEST_ROOT/foo" --dry-run | grepQuiet "experimental Nix feature 'mounted-ssh-store' is disabled" + enableFeatures mounted-ssh-store +# N.B. encoded query param is `mounted={}`. In the future, we can just +# do `--store` with JSON, and then the nested structure will actually +# bring benefits. nix build -Lvf simple.nix \ --arg busybox "$busybox" \ --out-link "$TEST_ROOT/result-from-remote" \ - --store mounted-ssh-ng://localhost + --store 'ssh-ng://localhost?mounted=%7B%7D' nix build -Lvf simple.nix \ --arg busybox "$busybox" \ --out-link "$TEST_ROOT/result-from-remote-new-cli" \ - --store 'mounted-ssh-ng://localhost?remote-program=nix daemon' + --store 'ssh-ng://localhost?mounted=%7B%7D&remote-program=nix daemon' # This verifies that the out link was actually created and valid. The ability # to create out links (permanent gc roots) is the distinguishing feature of