From 58b29671150de4a587ce912d4307d5f5e7b09f8a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 23 May 2025 15:29:55 -0400 Subject: [PATCH 1/3] More flexible typing for `get` in `util.hh` Co-authored-by: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> --- src/libutil/include/nix/util/util.hh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/libutil/include/nix/util/util.hh b/src/libutil/include/nix/util/util.hh index 56041a112e0..077bfb231bd 100644 --- a/src/libutil/include/nix/util/util.hh +++ b/src/libutil/include/nix/util/util.hh @@ -196,19 +196,19 @@ std::pair getLine(std::string_view s); /** * Get a value for the specified key from an associate container. */ -template -const typename T::mapped_type * get(const T & map, const typename T::key_type & key) +template +const typename T::mapped_type * get(const T & map, K && key) { - auto i = map.find(key); + auto i = map.find(std::forward(key)); if (i == map.end()) return nullptr; return &i->second; } -template -typename T::mapped_type * get(T & map, const typename T::key_type & key) +template +typename T::mapped_type * get(T & map, K && key) { - auto i = map.find(key); + auto i = map.find(std::forward(key)); if (i == map.end()) return nullptr; return &i->second; @@ -217,11 +217,10 @@ typename T::mapped_type * get(T & map, const typename T::key_type & key) /** * Get a value for the specified key from an associate container, or a default value if the key isn't present. */ -template -const typename T::mapped_type & -getOr(T & map, const typename T::key_type & key, const typename T::mapped_type & defaultValue) +template +const typename T::mapped_type & getOr(T & map, K && key, const typename T::mapped_type & defaultValue) { - auto i = map.find(key); + auto i = map.find(std::forward(key)); if (i == map.end()) return defaultValue; return i->second; From a51537f31cd5bfdc6f892daa5e3f9c06416ef866 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Mar 2025 11:29:06 -0400 Subject: [PATCH 2/3] New store settings system Motivation: See the linked issues for details. The most notable user-relevant bits are: - This cleans up the `MountedSSHStore`: decomposed into its orthogonal parts - This brings us pretty close to being able to then implement a JSON-based config. - Store query parameters can be JSON - Stores can entirely be specified via JSON objects, but this is not yet hooked up to anything. Also behind the scenes have these benefits: 1. The docs are moved out of the headers, good for less rebuilding when they changes 2. Stores are always constructed from store configs 3. Use JSON, avoid custom serializers Context: Part of #11106 Co-Authored-By: Robert Hensing Co-authored-by: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> --- doc/manual/generate-settings.nix | 46 +++++- .../data/store-reference/auto.json | 3 + .../data/store-reference/auto_param.json | 4 + .../data/store-reference/local_1.json | 5 + .../data/store-reference/local_2.json | 5 + .../data/store-reference/ssh.json | 4 + .../data/store-reference/unix.json | 6 + src/libstore-tests/dummy-store.cc | 20 +++ src/libstore-tests/legacy-ssh-store.cc | 6 +- src/libstore-tests/local-store.cc | 6 - src/libstore-tests/meson.build | 1 + src/libstore-tests/ssh-store.cc | 70 ++++++-- src/libstore-tests/store-reference.cc | 53 ++++-- src/libstore/binary-cache-store.cc | 112 +++++++++++-- src/libstore/common-ssh-store-config.cc | 54 ++++++- src/libstore/config-parse.cc | 71 ++++++++ src/libstore/dummy-store.cc | 57 +++---- src/libstore/filetransfer.cc | 2 +- src/libstore/http-binary-cache-store.cc | 35 ++-- .../include/nix/store/binary-cache-store.hh | 77 +++------ .../nix/store/common-ssh-store-config.hh | 38 ++--- .../include/nix/store/config-parse-impl.hh | 69 ++++++++ .../include/nix/store/config-parse.hh | 145 +++++++++++++++++ src/libstore/include/nix/store/dummy-store.hh | 26 +++ .../nix/store/http-binary-cache-store.hh | 7 +- .../include/nix/store/legacy-ssh-store.hh | 30 ++-- .../nix/store/local-binary-cache-store.hh | 6 +- .../include/nix/store/local-fs-store.hh | 36 ++--- .../include/nix/store/local-overlay-store.hh | 73 ++------- src/libstore/include/nix/store/local-store.hh | 75 +++------ src/libstore/include/nix/store/machines.hh | 2 + src/libstore/include/nix/store/meson.build | 3 + .../include/nix/store/remote-store.hh | 20 +-- .../nix/store/s3-binary-cache-store.hh | 108 +++---------- src/libstore/include/nix/store/ssh-store.hh | 49 ++---- src/libstore/include/nix/store/store-api.hh | 150 ++++++++--------- .../include/nix/store/store-dir-config.hh | 59 ++++--- .../include/nix/store/store-reference.hh | 18 ++- .../include/nix/store/store-registration.hh | 31 ++-- .../include/nix/store/uds-remote-store.hh | 19 +-- src/libstore/legacy-ssh-store.cc | 53 +++++- src/libstore/local-binary-cache-store.cc | 18 ++- src/libstore/local-fs-store.cc | 84 ++++++++-- src/libstore/local-overlay-store.cc | 90 ++++++++++- src/libstore/local-store.cc | 93 ++++++++++- src/libstore/machines.cc | 19 +-- src/libstore/meson.build | 1 + src/libstore/mounted-ssh-store.md | 18 --- src/libstore/remote-store.cc | 45 +++++- src/libstore/s3-binary-cache-store.cc | 138 +++++++++++++--- src/libstore/ssh-store.cc | 129 ++++++++++----- src/libstore/ssh-store.md | 1 - src/libstore/store-api.cc | 93 +++++++++++ src/libstore/store-dir-config.cc | 38 ++++- src/libstore/store-reference.cc | 151 +++++++++++++++++- src/libstore/store-registration.cc | 35 ++-- src/libstore/uds-remote-store.cc | 24 +-- src/libstore/unix/build/derivation-builder.cc | 6 +- .../include/nix/util/config-abstract.hh | 45 ++++++ src/libutil/include/nix/util/configuration.hh | 3 +- src/libutil/include/nix/util/meson.build | 1 + src/nix/main.cc | 2 +- src/nix/unix/daemon.cc | 4 +- .../build-remote-with-mounted-ssh-ng.sh | 12 +- 64 files changed, 1965 insertions(+), 739 deletions(-) create mode 100644 src/libstore-tests/data/store-reference/auto.json create mode 100644 src/libstore-tests/data/store-reference/auto_param.json create mode 100644 src/libstore-tests/data/store-reference/local_1.json create mode 100644 src/libstore-tests/data/store-reference/local_2.json create mode 100644 src/libstore-tests/data/store-reference/ssh.json create mode 100644 src/libstore-tests/data/store-reference/unix.json create mode 100644 src/libstore-tests/dummy-store.cc create mode 100644 src/libstore/config-parse.cc create mode 100644 src/libstore/include/nix/store/config-parse-impl.hh create mode 100644 src/libstore/include/nix/store/config-parse.hh create mode 100644 src/libstore/include/nix/store/dummy-store.hh delete mode 100644 src/libstore/mounted-ssh-store.md create mode 100644 src/libutil/include/nix/util/config-abstract.hh diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 35ae73e5d1f..bf94c24e286 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -19,7 +19,6 @@ in prefix, inlineHTML ? true, }: -settingsInfo: let @@ -27,11 +26,25 @@ let prefix: setting: { description, - documentDefault, - defaultValue, - aliases, - value, + experimentalFeature, + + # Whether we document the default, because it is machine agostic, + # or don't because because it is machine-specific. + documentDefault ? true, + + # The default value is JSON for new-style config, rather than then + # a string or boolean, for old-style config. + isJson ? false, + + defaultValue ? null, + + subSettings ? null, + + aliases ? [ ], + + # The current value for this setting. Purposefully unused. + value ? null, }: let result = squash '' @@ -50,7 +63,7 @@ let ${description} - **Default:** ${showDefault documentDefault defaultValue} + ${showDefaultOrSubSettings} ${showAliases aliases} ''; @@ -72,9 +85,24 @@ let > ``` ''; + showDefaultOrSubSettings = + if !isAttrs subSettings then + # No subsettings, instead single setting. Show the default value. + '' + **Default:** ${showDefault} + '' + else + # Indent the nested sub-settings, and append the outer setting name onto the prefix + indent " " '' + **Nullable sub-settings**: ${if subSettings.nullable then "true" else "false"} + ${builtins.trace prefix (showSettings "${prefix}-${setting}" subSettings.map)} + ''; + showDefault = - documentDefault: defaultValue: if documentDefault then + if isJson then + "`${builtins.toJSON defaultValue}`" + else # a StringMap value type is specified as a string, but # this shows the value type. The empty stringmap is `null` in # JSON, but that converts to `{ }` here. @@ -95,5 +123,7 @@ let in result; + showSettings = + prefix: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo)); in -concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo)) +showSettings prefix diff --git a/src/libstore-tests/data/store-reference/auto.json b/src/libstore-tests/data/store-reference/auto.json new file mode 100644 index 00000000000..da510f64949 --- /dev/null +++ b/src/libstore-tests/data/store-reference/auto.json @@ -0,0 +1,3 @@ +{ + "scheme": "auto" +} diff --git a/src/libstore-tests/data/store-reference/auto_param.json b/src/libstore-tests/data/store-reference/auto_param.json new file mode 100644 index 00000000000..41826c83b56 --- /dev/null +++ b/src/libstore-tests/data/store-reference/auto_param.json @@ -0,0 +1,4 @@ +{ + "root": "/foo/bar/baz", + "scheme": "auto" +} diff --git a/src/libstore-tests/data/store-reference/local_1.json b/src/libstore-tests/data/store-reference/local_1.json new file mode 100644 index 00000000000..3b44e382b7f --- /dev/null +++ b/src/libstore-tests/data/store-reference/local_1.json @@ -0,0 +1,5 @@ +{ + "authority": "", + "root": "/foo/bar/baz", + "scheme": "local" +} diff --git a/src/libstore-tests/data/store-reference/local_2.json b/src/libstore-tests/data/store-reference/local_2.json new file mode 100644 index 00000000000..346b0cf225f --- /dev/null +++ b/src/libstore-tests/data/store-reference/local_2.json @@ -0,0 +1,5 @@ +{ + "authority": "/foo/bar/baz", + "scheme": "local", + "trusted": true +} diff --git a/src/libstore-tests/data/store-reference/ssh.json b/src/libstore-tests/data/store-reference/ssh.json new file mode 100644 index 00000000000..e5b0b367ffd --- /dev/null +++ b/src/libstore-tests/data/store-reference/ssh.json @@ -0,0 +1,4 @@ +{ + "authority": "localhost", + "scheme": "ssh" +} diff --git a/src/libstore-tests/data/store-reference/unix.json b/src/libstore-tests/data/store-reference/unix.json new file mode 100644 index 00000000000..8943e959dd6 --- /dev/null +++ b/src/libstore-tests/data/store-reference/unix.json @@ -0,0 +1,6 @@ +{ + "authority": "", + "max-connections": 7, + "scheme": "unix", + "trusted": true +} diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc new file mode 100644 index 00000000000..655166d38c3 --- /dev/null +++ b/src/libstore-tests/dummy-store.cc @@ -0,0 +1,20 @@ +#include + +#include "nix/store/dummy-store.hh" +#include "nix/store/globals.hh" + +namespace nix { + +TEST(DummyStore, constructConfig) +{ + DummyStoreConfig config{"dummy", "", {}}; + + EXPECT_EQ(config.storeDir, settings.nixStore); +} + +TEST(DummyStore, constructConfigNoAuthority) +{ + EXPECT_THROW(DummyStoreConfig("dummy", "not-allowed", {}), UsageError); +} + +} // namespace nix diff --git a/src/libstore-tests/legacy-ssh-store.cc b/src/libstore-tests/legacy-ssh-store.cc index 04c3763ec09..cf8adb5f4ea 100644 --- a/src/libstore-tests/legacy-ssh-store.cc +++ b/src/libstore-tests/legacy-ssh-store.cc @@ -14,8 +14,10 @@ TEST(LegacySSHStore, constructConfig) StoreConfig::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, }, }); diff --git a/src/libstore-tests/local-store.cc b/src/libstore-tests/local-store.cc index cdbc29b0319..6d9009825a4 100644 --- a/src/libstore-tests/local-store.cc +++ b/src/libstore-tests/local-store.cc @@ -2,12 +2,6 @@ #include "nix/store/local-store.hh" -// Needed for template specialisations. This is not good! When we -// overhaul how store configs work, this should be fixed. -#include "nix/util/args.hh" -#include "nix/util/config-impl.hh" -#include "nix/util/abstract-setting-to-json.hh" - namespace nix { TEST(LocalStore, constructConfig_rootQueryParam) diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index e5995bcb127..12e0175d291 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -60,6 +60,7 @@ sources = files( 'derivation.cc', 'derived-path.cc', 'downstream-placeholder.cc', + 'dummy-store.cc', 'http-binary-cache-store.cc', 'legacy-ssh-store.cc', 'local-binary-cache-store.cc', diff --git a/src/libstore-tests/ssh-store.cc b/src/libstore-tests/ssh-store.cc index 335e4ae850d..4dffbfb8180 100644 --- a/src/libstore-tests/ssh-store.cc +++ b/src/libstore-tests/ssh-store.cc @@ -1,8 +1,6 @@ #include #include "nix/store/ssh-store.hh" -#include "nix/util/config-impl.hh" -#include "nix/util/abstract-setting-to-json.hh" namespace nix { @@ -14,8 +12,10 @@ TEST(SSHStore, constructConfig) StoreConfig::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, }, }, }; @@ -28,22 +28,32 @@ TEST(SSHStore, constructConfig) })); EXPECT_EQ(config.getUri(), "ssh-ng://me@localhost:2222?remote-program=foo%20bar"); - config.resetOverridden(); + config.authority.port = std::nullopt; EXPECT_EQ(config.getUri(), "ssh-ng://me@localhost:2222"); } TEST(MountedSSHStore, constructConfig) { - MountedSSHStoreConfig config{ - "mounted-ssh", + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "mounted-ssh-store"); + + SSHStoreConfig config{ + "ssh-ng", "localhost", StoreConfig::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, + }, + { + "mounted", + nlohmann::json::object_t{}, }, }, + mockXpSettings, }; EXPECT_EQ( @@ -52,6 +62,48 @@ TEST(MountedSSHStore, constructConfig) "foo", "bar", })); + + ASSERT_TRUE(config.mounted); + + EXPECT_EQ(config.mounted->realStoreDir, "/nix/store"); +} + +TEST(MountedSSHStore, constructConfigWithFunnyRealStoreDir) +{ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "mounted-ssh-store"); + + SSHStoreConfig config{ + "ssh-ng", + "localhost", + StoreConfig::Params{ + { + "remote-program", + { + "foo", + "bar", + }, + }, + { + "mounted", + nlohmann::json::object_t{ + {"real", "/foo/bar"}, + }, + }, + }, + mockXpSettings, + }; + + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); + + ASSERT_TRUE(config.mounted); + + EXPECT_EQ(config.mounted->realStoreDir, "/foo/bar"); } } // namespace nix diff --git a/src/libstore-tests/store-reference.cc b/src/libstore-tests/store-reference.cc index 01b75f3d264..1ee10c2816d 100644 --- a/src/libstore-tests/store-reference.cc +++ b/src/libstore-tests/store-reference.cc @@ -17,14 +17,14 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest std::filesystem::path goldenMaster(PathView testStem) const override { - return unitTestData / (testStem + ".txt"); + return unitTestData / testStem; } }; #define URI_TEST_READ(STEM, OBJ) \ TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_uri) \ { \ - readTest(#STEM, ([&](const auto & encoded) { \ + readTest(#STEM ".txt", ([&](const auto & encoded) { \ StoreReference expected = OBJ; \ auto got = StoreReference::parse(encoded); \ ASSERT_EQ(got, expected); \ @@ -35,7 +35,7 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_uri) \ { \ writeTest( \ - #STEM, \ + #STEM ".txt", \ [&]() -> StoreReference { return OBJ; }, \ [](const auto & file) { return StoreReference::parse(readFile(file)); }, \ [](const auto & file, const auto & got) { return writeFile(file, got.render()); }); \ @@ -45,14 +45,43 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest URI_TEST_READ(STEM, OBJ) \ URI_TEST_WRITE(STEM, OBJ) -URI_TEST( +#define JSON_TEST_READ(STEM, OBJ) \ + TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_json) \ + { \ + readTest(#STEM ".json", ([&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + StoreReference expected = OBJ; \ + StoreReference got = encoded; \ + ASSERT_EQ(got, expected); \ + })); \ + } + +#define JSON_TEST_WRITE(STEM, OBJ) \ + TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_json) \ + { \ + writeTest( \ + #STEM ".json", \ + [&]() -> StoreReference { return OBJ; }, \ + [](const auto & file) -> StoreReference { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, json(got).dump(2) + "\n"); }); \ + } + +#define JSON_TEST(STEM, OBJ) \ + JSON_TEST_READ(STEM, OBJ) \ + JSON_TEST_WRITE(STEM, OBJ) + +#define BOTH_FORMATS_TEST(STEM, OBJ) \ + URI_TEST(STEM, OBJ) \ + JSON_TEST(STEM, OBJ) + +BOTH_FORMATS_TEST( auto, (StoreReference{ .variant = StoreReference::Auto{}, .params = {}, })) -URI_TEST( +BOTH_FORMATS_TEST( auto_param, (StoreReference{ .variant = StoreReference::Auto{}, @@ -81,7 +110,7 @@ static StoreReference localExample_2{ }, .params = { - {"trusted", "true"}, + {"trusted", true}, }, }; @@ -96,9 +125,9 @@ static StoreReference localExample_3{ }, }; -URI_TEST(local_1, localExample_1) +BOTH_FORMATS_TEST(local_1, localExample_1) -URI_TEST(local_2, localExample_2) +BOTH_FORMATS_TEST(local_2, localExample_2) /* Test path with spaces */ URI_TEST(local_3, localExample_3) @@ -114,16 +143,16 @@ static StoreReference unixExample{ }, .params = { - {"max-connections", "7"}, - {"trusted", "true"}, + {"max-connections", 7}, + {"trusted", true}, }, }; -URI_TEST(unix, unixExample) +BOTH_FORMATS_TEST(unix, unixExample) URI_TEST_READ(unix_shorthand, unixExample) -URI_TEST( +BOTH_FORMATS_TEST( ssh, (StoreReference{ .variant = diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index c55239413a0..14378e60f53 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -13,6 +13,7 @@ #include "nix/util/callback.hh" #include "nix/util/signals.hh" #include "nix/util/archive.hh" +#include "nix/store/config-parse-impl.hh" #include #include @@ -24,23 +25,104 @@ namespace nix { -BinaryCacheStore::BinaryCacheStore(Config & config) - : config{config} +constexpr static const BinaryCacheStoreConfigT binaryCacheStoreConfigDescriptions = { + .compression = + { + .name = "compression", + .description = "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`).", + }, + .writeNARListing = + { + .name = "write-nar-listing", + .description = "Whether to write a JSON file that lists the files in each NAR.", + }, + .writeDebugInfo = + { + .name = "index-debug-info", + .description = R"( + Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to + fetch debug info on demand + )", + }, + .secretKeyFile{ + .name = "secret-key", + .description = "Path to the secret key used to sign the binary cache.", + }, + .localNarCache{ + .name = "local-nar-cache", + .description = + "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`.", + }, + .parallelCompression{ + .name = "parallel-compression", + .description = + "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`.", + }, + .compressionLevel{ + .name = "compression-level", + .description = R"( + The *preset level* to be used when compressing NARs. + The meaning and accepted values depend on the compression method selected. + `-1` specifies that the default compression level should be used. + )", + }, +}; + +#define BINARY_CACHE_STORE_CONFIG_FIELDS(X) \ + X(compression), X(writeNARListing), X(writeDebugInfo), X(secretKeyFile), X(secretKeyFiles), X(localNarCache), \ + X(parallelCompression), X(compressionLevel), + +MAKE_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS) + +static BinaryCacheStoreConfigT binaryCacheStoreConfigDefaults() +{ + return { + .compression = {"xz"}, + .writeNARListing = {false}, + .writeDebugInfo = {false}, + .secretKeyFile = {""}, + .secretKeyFiles = {{}}, + .localNarCache = {""}, + .parallelCompression = {false}, + .compressionLevel = {-1}, + }; +} + +MAKE_APPLY_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS) + +BinaryCacheStore::Config::BinaryCacheStoreConfig(const Store::Config & storeConfig, const StoreConfig::Params & params) + : BinaryCacheStoreConfigT{binaryCacheStoreConfigApplyParse(params)} + , storeConfig{storeConfig} +{ +} + +config::SettingDescriptionMap BinaryCacheStoreConfig::descriptions() +{ + constexpr auto & descriptions = binaryCacheStoreConfigDescriptions; + auto defaults = binaryCacheStoreConfigDefaults(); + return {BINARY_CACHE_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}; +} + +BinaryCacheStore::BinaryCacheStore(const Config & config) + : Store{config.storeConfig} + , config{config} { if (config.secretKeyFile != "") signers.push_back(std::make_unique(SecretKey{readFile(config.secretKeyFile)})); - if (config.secretKeyFiles != "") { - std::stringstream ss(config.secretKeyFiles); - Path keyPath; - while (std::getline(ss, keyPath, ',')) { - signers.push_back(std::make_unique(SecretKey{readFile(keyPath)})); - } + for (auto & keyPath : config.secretKeyFiles.value) { + signers.push_back(std::make_unique(SecretKey{readFile(keyPath)})); } StringSink sink; sink << narVersionMagic1; narMagic = sink.s; + + // Want to call this but cannot, because virtual function lookup is + // disabled in a constructor. It is thus left to instances to call + // it instead. + + // init(); } void BinaryCacheStore::init() @@ -59,13 +141,13 @@ void BinaryCacheStore::init() if (value != storeDir) throw Error( "binary cache '%s' is for Nix stores with prefix '%s', not '%s'", - config.getUri(), + config.storeConfig.getUri(), value, storeDir); } else if (name == "WantMassQuery") { - config.wantMassQuery.setDefault(value == "1"); + resolvedSubstConfig.wantMassQuery.value = config.storeConfig.wantMassQuery.value_or(value == "1"); } else if (name == "Priority") { - config.priority.setDefault(std::stoi(value)); + resolvedSubstConfig.priority.value = config.storeConfig.priority.value_or(std::stoi(value)); } } } @@ -133,7 +215,7 @@ void BinaryCacheStore::writeNarInfo(ref narInfo) if (diskCache) diskCache->upsertNarInfo( - config.getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr(narInfo)); + config.storeConfig.getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr(narInfo)); } ref BinaryCacheStore::addToStoreCommon( @@ -431,7 +513,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) void BinaryCacheStore::queryPathInfoUncached( const StorePath & storePath, Callback> callback) noexcept { - auto uri = config.getUri(); + auto uri = config.storeConfig.getUri(); auto storePathS = printStorePath(storePath); auto act = std::make_shared( *logger, @@ -531,7 +613,7 @@ void BinaryCacheStore::queryRealisationUncached( void BinaryCacheStore::registerDrvOutput(const Realisation & info) { if (diskCache) - diskCache->upsertRealisation(config.getUri(), info); + diskCache->upsertRealisation(config.storeConfig.getUri(), info); auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; upsertFile(filePath, info.toJSON().dump(), "application/json"); } @@ -559,7 +641,7 @@ std::optional BinaryCacheStore::getBuildLogExact(const StorePath & { auto logPath = "log/" + std::string(baseNameOf(printStorePath(path))); - debug("fetching build log from binary cache '%s/%s'", config.getUri(), logPath); + debug("fetching build log from binary cache '%s/%s'", config.storeConfig.getUri(), logPath); return getFile(logPath); } diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index 12f187b4c9e..6ab6fb33229 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -2,17 +2,65 @@ #include "nix/store/common-ssh-store-config.hh" #include "nix/store/ssh.hh" +#include "nix/store/config-parse-impl.hh" namespace nix { -CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) +constexpr static const CommonSSHStoreConfigT commonSSHStoreConfigDescriptions = { + .sshKey{ + .name = "ssh-key", + .description = "Path to the SSH private key used to authenticate to the remote machine.", + }, + .sshPublicHostKey{ + .name = "base64-ssh-public-host-key", + .description = "The public host key of the remote machine.", + }, + .compress{ + .name = "compress", + .description = "Whether to enable SSH compression.", + }, + .remoteStore{ + .name = "remote-store", + .description = R"( + [Store URL](@docroot@/store/types/index.md#store-url-format) + to be used on the remote machine. The default is `auto` + (i.e. use the Nix daemon or `/nix/store` directly). + )", + }, +}; + +#define COMMON_SSH_STORE_CONFIG_FIELDS(X) X(sshKey), X(sshPublicHostKey), X(compress), X(remoteStore), + +MAKE_PARSE(CommonSSHStoreConfig, commonSSHStoreConfig, COMMON_SSH_STORE_CONFIG_FIELDS) + +static CommonSSHStoreConfigT commonSSHStoreConfigDefaults() +{ + return { + .sshKey = {""}, + .sshPublicHostKey = {""}, + .compress = {false}, + .remoteStore = {""}, + }; +} + +MAKE_APPLY_PARSE(CommonSSHStoreConfig, commonSSHStoreConfig, COMMON_SSH_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap CommonSSHStoreConfig::descriptions() +{ + constexpr auto & descriptions = commonSSHStoreConfigDescriptions; + auto defaults = commonSSHStoreConfigDefaults(); + return {COMMON_SSH_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}; +} + +CommonSSHStoreConfig::CommonSSHStoreConfig( + std::string_view scheme, std::string_view authority, const StoreConfig::Params & params) : CommonSSHStoreConfig(scheme, ParsedURL::Authority::parse(authority), params) { } CommonSSHStoreConfig::CommonSSHStoreConfig( - std::string_view scheme, const ParsedURL::Authority & authority, const Params & params) - : StoreConfig(params) + std::string_view scheme, const ParsedURL::Authority & authority, const StoreConfig::Params & params) + : CommonSSHStoreConfigT{commonSSHStoreConfigApplyParse(params)} , authority(authority) { } diff --git a/src/libstore/config-parse.cc b/src/libstore/config-parse.cc new file mode 100644 index 00000000000..f54245a142e --- /dev/null +++ b/src/libstore/config-parse.cc @@ -0,0 +1,71 @@ +#include + +#include "nix/store/config-parse.hh" +#include "nix/util/json-utils.hh" +#include "nix/util/util.hh" + +namespace nix::config { + +}; + +namespace nlohmann { + +using namespace nix::config; + +SettingDescription adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return { + .description = getString(valueAt(obj, "description")), + .experimentalFeature = valueAt(obj, "experimentalFeature").get>(), + .info = [&]() -> decltype(SettingDescription::info) { + if (auto documentDefault = optionalValueAt(obj, "documentDefault")) { + return SettingDescription::Single{ + .defaultValue = *documentDefault ? (std::optional{valueAt(obj, "defaultValue")}) + : (std::optional{}), + }; + } else { + auto & subObj = getObject(valueAt(obj, "subSettings")); + return SettingDescription::Sub{ + .nullable = valueAt(subObj, "nullable"), + .map = valueAt(subObj, "map"), + }; + } + }(), + }; +} + +void adl_serializer::to_json(json & obj, SettingDescription sd) +{ + obj.emplace("description", sd.description); + // obj.emplace("aliases", sd.aliases); + obj.emplace("experimentalFeature", sd.experimentalFeature); + + std::visit( + overloaded{ + [&](const SettingDescription::Single & single) { + // Indicate the default value is JSON, rather than a legacy setting + // boolean or string. + // + // TODO remove if we no longer have the legacy setting system / the + // code handling doc rendering of the settings is decoupled. + obj.emplace("isJson", true); + + // Cannot just use `null` because the default value might itself be + // `null`. + obj.emplace("documentDefault", single.defaultValue.has_value()); + + if (single.defaultValue.has_value()) + obj.emplace("defaultValue", *single.defaultValue); + }, + [&](const SettingDescription::Sub & sub) { + json subJson; + subJson.emplace("nullable", sub.nullable); + subJson.emplace("map", sub.map); + obj.emplace("subSettings", std::move(subJson)); + }, + }, + sd.info); +} + +} // namespace nlohmann diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index d0e2989681a..908780eb66a 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -1,48 +1,33 @@ +#include "nix/store/dummy-store.hh" #include "nix/store/store-registration.hh" #include "nix/util/callback.hh" namespace nix { -struct DummyStoreConfig : public std::enable_shared_from_this, virtual StoreConfig +DummyStoreConfig::DummyStoreConfig( + std::string_view scheme, std::string_view authority, const StoreConfig::Params & params) + : StoreConfig{params} { - using StoreConfig::StoreConfig; - - DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - { - if (!authority.empty()) - throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); - } - - static const std::string name() - { - return "Dummy Store"; - } + if (!authority.empty()) + throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); +} - static std::string doc() - { - return +std::string DummyStoreConfig::doc() +{ + return #include "dummy-store.md" - ; - } - - static StringSet uriSchemes() - { - return {"dummy"}; - } - - ref openStore() const override; + ; +} - StoreReference getReference() const override - { - return { - .variant = - StoreReference::Specified{ - .scheme = *uriSchemes().begin(), - }, - }; - } -}; +StoreReference DummyStoreConfig::getReference() const +{ + return { + .variant = + StoreReference::Specified{ + .scheme = *uriSchemes().begin(), + }, + }; +} struct DummyStore : virtual Store { diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index c29da12e8e5..2f31138ea9d 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -799,7 +799,7 @@ struct curlFileTransfer : public FileTransfer } #if NIX_WITH_S3_SUPPORT - std::tuple parseS3Uri(std::string uri) + std::tuple parseS3Uri(std::string uri) { auto [path, params] = splitUriAndParams(uri); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index fc19bc1f8e1..a79b14c84d2 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -7,6 +7,14 @@ namespace nix { +config::SettingDescriptionMap HttpBinaryCacheStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(BinaryCacheStoreConfig::descriptions()); + return ret; +} + MakeError(UploadToHTTP, Error); StringSet HttpBinaryCacheStoreConfig::uriSchemes() @@ -19,9 +27,9 @@ StringSet HttpBinaryCacheStoreConfig::uriSchemes() } HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( - std::string_view scheme, std::string_view _cacheUri, const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) + std::string_view scheme, std::string_view _cacheUri, const StoreConfig::Params & params) + : Store::Config{params} + , BinaryCacheStoreConfig{*this, params} , cacheUri(parseURL( std::string{scheme} + "://" + (!_cacheUri.empty() ? _cacheUri @@ -29,6 +37,7 @@ HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( { while (!cacheUri.path.empty() && cacheUri.path.back() == '/') cacheUri.path.pop_back(); + assert(cacheUri.query.empty()); } StoreReference HttpBinaryCacheStoreConfig::getReference() const @@ -39,7 +48,6 @@ StoreReference HttpBinaryCacheStoreConfig::getReference() const .scheme = cacheUri.scheme, .authority = (cacheUri.authority ? cacheUri.authority->to_string() : "") + cacheUri.path, }, - .params = cacheUri.query, }; } @@ -64,22 +72,25 @@ class HttpBinaryCacheStore : public virtual BinaryCacheStore using Config = HttpBinaryCacheStoreConfig; - ref config; + ref config; - HttpBinaryCacheStore(ref config) - : Store{*config} // TODO it will actually mutate the configuration + HttpBinaryCacheStore(ref config) + : Store{*config} , BinaryCacheStore{*config} , config{config} { diskCache = getNarInfoDiskCache(); + + init(); } void init() override { // FIXME: do this lazily? if (auto cacheInfo = diskCache->upToDateCacheExists(config->cacheUri.to_string())) { - config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); - config->priority.setDefault(cacheInfo->priority); + resolvedSubstConfig.wantMassQuery.value = + config->storeConfig.wantMassQuery.value_or(cacheInfo->wantMassQuery); + resolvedSubstConfig.priority.value = config->storeConfig.priority.value_or(cacheInfo->priority); } else { try { BinaryCacheStore::init(); @@ -87,7 +98,7 @@ class HttpBinaryCacheStore : public virtual BinaryCacheStore throw Error("'%s' does not appear to be a binary cache", config->cacheUri.to_string()); } diskCache->createCache( - config->cacheUri.to_string(), config->storeDir, config->wantMassQuery, config->priority); + config->cacheUri.to_string(), storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority); } } @@ -244,9 +255,7 @@ class HttpBinaryCacheStore : public virtual BinaryCacheStore ref HttpBinaryCacheStore::Config::openStore() const { - return make_ref( - ref{// FIXME we shouldn't actually need a mutable config - std::const_pointer_cast(shared_from_this())}); + return make_ref(ref{shared_from_this()}); } static RegisterStoreImplementation regHttpBinaryCacheStore; diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 908500b4280..3e5e4a73bfa 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -13,51 +13,26 @@ namespace nix { struct NarInfo; -struct BinaryCacheStoreConfig : virtual StoreConfig +template class F> +struct BinaryCacheStoreConfigT { - using StoreConfig::StoreConfig; - - const Setting compression{ - this, "xz", "compression", "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; - - const Setting writeNARListing{ - this, false, "write-nar-listing", "Whether to write a JSON file that lists the files in each NAR."}; - - const Setting writeDebugInfo{ - this, - false, - "index-debug-info", - R"( - Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to - fetch debug info on demand - )"}; - - const Setting secretKeyFile{this, "", "secret-key", "Path to the secret key used to sign the binary cache."}; - - const Setting secretKeyFiles{ - this, "", "secret-keys", "List of comma-separated paths to the secret keys used to sign the binary cache."}; - - const Setting localNarCache{ - this, - "", - "local-nar-cache", - "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; - - const Setting parallelCompression{ - this, - false, - "parallel-compression", - "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; - - const Setting compressionLevel{ - this, - -1, - "compression-level", - R"( - The *preset level* to be used when compressing NARs. - The meaning and accepted values depend on the compression method selected. - `-1` specifies that the default compression level should be used. - )"}; + F compression; + F writeNARListing; + F writeDebugInfo; + F secretKeyFile; + F> secretKeyFiles; + F localNarCache; + F parallelCompression; + F compressionLevel; +}; + +struct BinaryCacheStoreConfig : BinaryCacheStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); + + const Store::Config & storeConfig; + + BinaryCacheStoreConfig(const Store::Config &, const StoreConfig::Params &); }; /** @@ -68,11 +43,7 @@ struct BinaryCacheStore : virtual Store, virtual LogStore { using Config = BinaryCacheStoreConfig; - /** - * Intentionally mutable because some things we update due to the - * cache's own (remote side) settings. - */ - Config & config; + const Config & config; private: std::vector> signers; @@ -84,7 +55,7 @@ protected: const std::string cacheInfoFile = "nix-cache-info"; - BinaryCacheStore(Config &); + BinaryCacheStore(const Config &); public: @@ -120,7 +91,11 @@ public: public: - virtual void init() override; + /** + * Perform any necessary effectful operation to make the store up and + * running + */ + virtual void init(); private: diff --git a/src/libstore/include/nix/store/common-ssh-store-config.hh b/src/libstore/include/nix/store/common-ssh-store-config.hh index bbd81835d4f..58e9cd3be5e 100644 --- a/src/libstore/include/nix/store/common-ssh-store-config.hh +++ b/src/libstore/include/nix/store/common-ssh-store-config.hh @@ -8,30 +8,26 @@ namespace nix { class SSHMaster; -struct CommonSSHStoreConfig : virtual StoreConfig +template class F> +struct CommonSSHStoreConfigT { - using StoreConfig::StoreConfig; - - CommonSSHStoreConfig(std::string_view scheme, const ParsedURL::Authority & authority, const Params & params); - CommonSSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); - - const Setting sshKey{ - this, "", "ssh-key", "Path to the SSH private key used to authenticate to the remote machine."}; - - const Setting sshPublicHostKey{ - this, "", "base64-ssh-public-host-key", "The public host key of the remote machine."}; + F sshKey; + F sshPublicHostKey; + F compress; + F remoteStore; +}; - const Setting compress{this, false, "compress", "Whether to enable SSH compression."}; +struct CommonSSHStoreConfig : CommonSSHStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); - const Setting remoteStore{ - this, - "", - "remote-store", - R"( - [Store URL](@docroot@/store/types/index.md#store-url-format) - to be used on the remote machine. The default is `auto` - (i.e. use the Nix daemon or `/nix/store` directly). - )"}; + /** + * @param scheme Note this isn't stored by this mix-in class, but + * just used for better error messages. + */ + CommonSSHStoreConfig( + std::string_view scheme, const ParsedURL::Authority & authority, const StoreConfig::Params & params); + CommonSSHStoreConfig(std::string_view scheme, std::string_view authority, const StoreConfig::Params & params); /** * Authority representing the SSH host to connect to. diff --git a/src/libstore/include/nix/store/config-parse-impl.hh b/src/libstore/include/nix/store/config-parse-impl.hh new file mode 100644 index 00000000000..27c63816bbc --- /dev/null +++ b/src/libstore/include/nix/store/config-parse-impl.hh @@ -0,0 +1,69 @@ +#pragma once +///@file + +#include + +#include "nix/store/config-parse.hh" +#include "nix/util/util.hh" +#include "nix/util/configuration.hh" + +namespace nix::config { + +template +std::optional +SettingInfo::parseConfig(const nlohmann::json::object_t & map, const ExperimentalFeatureSettings & xpSettings) const +{ + const nlohmann::json * p = get(map, name); + if (p && experimentalFeature) + xpSettings.require(*experimentalFeature); + return p ? (std::optional{p->get()}) : std::nullopt; +} + +template +std::pair SettingInfo::describe(const PlainValue & def) const +{ + return { + std::string{name}, + SettingDescription{ + .description = stripIndentation(description), + .experimentalFeature = experimentalFeature, + .info = + SettingDescription::Single{ + .defaultValue = documentDefault ? (std::optional{nlohmann::json(def.value)}) + : (std::optional{}), + }, + }, + }; +} + +/** + * Look up the setting's name in a map, falling back on the default if + * it does not exist. + */ +#define CONFIG_ROW(FIELD) .FIELD = descriptions.FIELD.parseConfig(params, xpSettings) + +#define APPLY_ROW(FIELD) .FIELD = {.value = parsed.FIELD.value_or(std::move(defaults.FIELD))} + +#define DESCRIBE_ROW(FIELD) \ + { \ + descriptions.FIELD.describe(defaults.FIELD), \ + } + +#define MAKE_PARSE(CAPITAL, LOWER, FIELDS) \ + static CAPITAL##T LOWER##Parse( \ + const StoreReference::Params & params, \ + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) \ + { \ + constexpr auto & descriptions = LOWER##Descriptions; \ + return {FIELDS(CONFIG_ROW)}; \ + } + +#define MAKE_APPLY_PARSE(CAPITAL, LOWER, FIELDS) \ + static CAPITAL##T LOWER##ApplyParse(const StoreReference::Params & params) \ + { \ + auto defaults = LOWER##Defaults(); \ + auto parsed = LOWER##Parse(params); \ + return {FIELDS(APPLY_ROW)}; \ + } + +} // namespace nix::config diff --git a/src/libstore/include/nix/store/config-parse.hh b/src/libstore/include/nix/store/config-parse.hh new file mode 100644 index 00000000000..0d5fb41960a --- /dev/null +++ b/src/libstore/include/nix/store/config-parse.hh @@ -0,0 +1,145 @@ +#pragma once +///@file + +#include + +#include "nix/util/config-abstract.hh" +#include "nix/util/json-impls.hh" +#include "nix/util/experimental-features.hh" + +namespace nix { + +struct ExperimentalFeatureSettings; + +}; + +namespace nix::config { + +struct SettingDescription; + +/** + * Typed version used as source of truth, and for operations like + * defaulting configurations. + * + * It is important that this type support `constexpr` values to avoid + * running into issues with static initialization order. + */ +template +struct SettingInfo +{ + /** + * Name of the setting, used when parsing configuration maps + */ + std::string_view name; + + /** + * Description of the setting. It is used just for documentation. + */ + std::string_view description; + +#if 0 + /** + * Other names of the setting also used when parsing configuration + * maps. This is useful for back-compat, etc. + */ + std::set aliases; +#endif + + /** + * `ExperimentalFeature` that must be enabled if the setting is + * allowed to be used + */ + std::optional experimentalFeature; + + /** + * Whether to document the default value. (Some defaults are + * system-specific and should not be documented.) + */ + bool documentDefault = true; + + /** + * Describe the setting as a key-value pair (name -> other info). + * The default value will be rendered to JSON if it is to be + * documented. + */ + std::pair describe(const PlainValue & def) const; + + std::optional + parseConfig(const nlohmann::json::object_t & map, const ExperimentalFeatureSettings & xpSettings) const; +}; + +struct SettingDescription; + +/** + * Map of setting names to descriptions of those settings. + */ +using SettingDescriptionMap = std::map; + +/** + * Untyped version used for rendering docs. This is not the source of + * truth, it is generated from the typed one. + * + * @note No `name` field because this is intended to be used as the value type + * of a map + */ +struct SettingDescription +{ + /** + * @see SettingInfo::description + */ + std::string description; + +#if 0 + /** + * @see SettingInfo::aliases + */ + StringSet aliases; +#endif + + /** + * @see SettingInfo::experimentalFeature + */ + std::optional experimentalFeature; + + /** + * A single leaf setting, to be optionally specified by arbitrary + * value (of some type) or left default. + */ + struct Single + { + /** + * Optional, for the `SettingInfo::documentDefault = false` case. + */ + std::optional defaultValue; + }; + + /** + * A nested settings object + */ + struct Sub + { + /** + * If `false`, this is just pure namespaceing. If `true`, we + * have a distinction between `null` and `{}`, meaning + * enabling/disabling the entire settings group. + */ + bool nullable = true; + + SettingDescriptionMap map; + }; + + /** + * Variant for `info` below + */ + using Info = std::variant; + + /** + * More information about this setting, depending on whether its the + * single leaf setting or subsettings case + */ + Info info; +}; + +} // namespace nix::config + +JSON_IMPL(config::SettingDescription) diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh new file mode 100644 index 00000000000..d95574cb408 --- /dev/null +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -0,0 +1,26 @@ +#include "nix/store/store-api.hh" + +namespace nix { + +struct DummyStoreConfig : std::enable_shared_from_this, StoreConfig +{ + DummyStoreConfig(std::string_view scheme, std::string_view authority, const StoreConfig::Params & params); + + static const std::string name() + { + return "Dummy Store"; + } + + static std::string doc(); + + static StringSet uriSchemes() + { + return {"dummy"}; + } + + ref openStore() const override; + + StoreReference getReference() const override; +}; + +} // namespace nix diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index e0f6ce42fdf..83784dde534 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -4,13 +4,12 @@ namespace nix { struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this, - virtual Store::Config, + Store::Config, BinaryCacheStoreConfig { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + static config::SettingDescriptionMap descriptions(); - HttpBinaryCacheStoreConfig( - std::string_view scheme, std::string_view cacheUri, const Store::Config::Params & params); + HttpBinaryCacheStoreConfig(std::string_view scheme, std::string_view cacheUri, const StoreConfig::Params & params); ParsedURL cacheUri; diff --git a/src/libstore/include/nix/store/legacy-ssh-store.hh b/src/libstore/include/nix/store/legacy-ssh-store.hh index 91e021433e5..eef89b99513 100644 --- a/src/libstore/include/nix/store/legacy-ssh-store.hh +++ b/src/libstore/include/nix/store/legacy-ssh-store.hh @@ -10,25 +10,27 @@ namespace nix { -struct LegacySSHStoreConfig : std::enable_shared_from_this, virtual CommonSSHStoreConfig +template class F> +struct LegacySSHStoreConfigT { - using CommonSSHStoreConfig::CommonSSHStoreConfig; + F remoteProgram; + F maxConnections; +}; - LegacySSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); +struct LegacySSHStoreConfig : std::enable_shared_from_this, + Store::Config, + CommonSSHStoreConfig, + LegacySSHStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); -#ifndef _WIN32 - // Hack for getting remote build log output. - // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in - // the documentation - const Setting logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"}; -#else + /** + * Hack for getting remote build log output. Intentionally not a + * documented user-visible setting. + */ Descriptor logFD = INVALID_DESCRIPTOR; -#endif - - const Setting remoteProgram{ - this, {"nix-store"}, "remote-program", "Path to the `nix-store` executable on the remote machine."}; - const Setting maxConnections{this, 1, "max-connections", "Maximum number of concurrent SSH connections."}; + LegacySSHStoreConfig(std::string_view scheme, std::string_view authority, const StoreConfig::Params & params); /** * Hack for hydra diff --git a/src/libstore/include/nix/store/local-binary-cache-store.hh b/src/libstore/include/nix/store/local-binary-cache-store.hh index 2846a9225c7..87e6ed375da 100644 --- a/src/libstore/include/nix/store/local-binary-cache-store.hh +++ b/src/libstore/include/nix/store/local-binary-cache-store.hh @@ -3,16 +3,16 @@ namespace nix { struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this, - virtual Store::Config, + Store::Config, BinaryCacheStoreConfig { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + static config::SettingDescriptionMap descriptions(); /** * @param binaryCacheDir `file://` is a short-hand for `file:///` * for now. */ - LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const Params & params); + LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const StoreConfig::Params & params); Path binaryCacheDir; diff --git a/src/libstore/include/nix/store/local-fs-store.hh b/src/libstore/include/nix/store/local-fs-store.hh index cae50e76259..c54c53e7a10 100644 --- a/src/libstore/include/nix/store/local-fs-store.hh +++ b/src/libstore/include/nix/store/local-fs-store.hh @@ -7,9 +7,22 @@ namespace nix { -struct LocalFSStoreConfig : virtual StoreConfig +template class F> +struct LocalFSStoreConfigT { - using StoreConfig::StoreConfig; + F> rootDir; + F stateDir; + F logDir; + F realStoreDir; +}; + +struct LocalFSStoreConfig : LocalFSStoreConfigT +{ + const Store::Config & storeConfig; + + static config::SettingDescriptionMap descriptions(); + + LocalFSStoreConfig(const Store::Config & storeConfig, const StoreConfig::Params &); /** * Used to override the `root` settings. Can't be done via modifying @@ -18,24 +31,7 @@ struct LocalFSStoreConfig : virtual StoreConfig * * @todo Make this less error-prone with new store settings system. */ - LocalFSStoreConfig(PathView path, const Params & params); - - OptionalPathSetting rootDir{this, std::nullopt, "root", "Directory prefixed to all other paths."}; - - PathSetting stateDir{ - this, - rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir, - "state", - "Directory where Nix stores state."}; - - PathSetting logDir{ - this, - rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir, - "log", - "directory where Nix stores log files."}; - - PathSetting realStoreDir{ - this, rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", "Physical path of the Nix store."}; + LocalFSStoreConfig(const Store::Config & storeConfig, PathView path, const StoreConfig::Params & params); }; struct LocalFSStore : virtual Store, virtual GcStore, virtual LogStore diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index b89d0a1a01a..2c5495f70e0 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -2,72 +2,23 @@ namespace nix { +template class F> +struct LocalOverlayStoreConfigT +{ + F> lowerStoreConfig; + F upperLayer; + F checkMount; + F remountHook; +}; + /** * Configuration for `LocalOverlayStore`. */ -struct LocalOverlayStoreConfig : virtual LocalStoreConfig +struct LocalOverlayStoreConfig : LocalStoreConfig, LocalOverlayStoreConfigT { - LocalOverlayStoreConfig(const StringMap & params) - : LocalOverlayStoreConfig("local-overlay", "", params) - { - } - - LocalOverlayStoreConfig(std::string_view scheme, PathView path, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(path, params) - , LocalStoreConfig(scheme, path, params) - { - } + static config::SettingDescriptionMap descriptions(); - const Setting lowerStoreUri{ - (StoreConfig *) this, - "", - "lower-store", - R"( - [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) - for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). - - Must be a store with a store dir on the file system. - Must be used as OverlayFS lower layer for this store's store dir. - )"}; - - const PathSetting upperLayer{ - (StoreConfig *) this, - "", - "upper-layer", - R"( - Directory containing the OverlayFS upper layer for this store's store dir. - )"}; - - Setting checkMount{ - (StoreConfig *) this, - true, - "check-mount", - R"( - Check that the overlay filesystem is correctly mounted. - - Nix does not manage the overlayfs mount point itself, but the correct - functioning of the overlay store does depend on this mount point being set up - correctly. Rather than just assume this is the case, check that the lowerdir - and upperdir options are what we expect them to be. This check is on by - default, but can be disabled if needed. - )"}; - - const PathSetting remountHook{ - (StoreConfig *) this, - "", - "remount-hook", - R"( - Script or other executable to run when overlay filesystem needs remounting. - - This is occasionally necessary when deleting a store path that exists in both upper and lower layers. - In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly - is the only way to perform the deletion without creating a "whiteout". - However this causes the OverlayFS kernel data structures to get out-of-sync, - and can lead to 'stale file handle' errors; remounting solves the problem. - - The store directory is passed as an argument to the invoked executable. - )"}; + LocalOverlayStoreConfig(std::string_view scheme, PathView path, const StoreConfig::Params & params); static const std::string name() { diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 3d7e8301a39..ae9d6b73909 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -31,72 +31,35 @@ struct OptimiseStats uint64_t bytesFreed = 0; }; -struct LocalBuildStoreConfig : virtual LocalFSStoreConfig +template class F> +struct LocalStoreConfigT { - -private: + F requireSigs; + F readOnly; /** Input for computing the build directory. See `getBuildDir()`. */ - Setting> buildDir{ - this, - std::nullopt, - "build-dir", - R"( - The directory on the host, in which derivations' temporary build directories are created. - - If not set, Nix will use the `builds` subdirectory of its configured state directory. - - Note that builds are often performed by the Nix daemon, so its `build-dir` applies. - - Nix will create this directory automatically with suitable permissions if it does not exist. - Otherwise its permissions must allow all users to traverse the directory (i.e. it must have `o+x` set, in unix parlance) for non-sandboxed builds to work correctly. - - This is also the location where [`--keep-failed`](@docroot@/command-ref/opt-common.md#opt-keep-failed) leaves its files. - - If Nix runs without sandbox, or if the platform does not support sandboxing with bind mounts (e.g. macOS), then the [`builder`](@docroot@/language/derivations.md#attr-builder)'s environment will contain this directory, instead of the virtual location [`sandbox-build-dir`](#conf-sandbox-build-dir). - - > **Warning** - > - > `build-dir` must not be set to a world-writable directory. - > Placing temporary build directories in a world-writable place allows other users to access or modify build data that is currently in use. - > This alone is merely an impurity, but combined with another factor this has allowed malicious derivations to escape the build sandbox. - )"}; -public: - Path getBuildDir() const; + F> buildDir; }; struct LocalStoreConfig : std::enable_shared_from_this, - virtual LocalFSStoreConfig, - virtual LocalBuildStoreConfig + Store::Config, + LocalFSStore::Config, + LocalStoreConfigT { - using LocalFSStoreConfig::LocalFSStoreConfig; + static config::SettingDescriptionMap descriptions(); - LocalStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); - - Setting requireSigs{ - this, - settings.requireSigs, - "require-sigs", - "Whether store paths copied into this store should have a trusted signature."}; - - Setting readOnly{ - this, - false, - "read-only", - R"( - Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. - - Normally Nix attempts to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. + LocalStoreConfig(const StoreConfig::Params & params) + : LocalStoreConfig{"local", "", params} + { + } - Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. + LocalStoreConfig(std::string_view scheme, std::string_view authority, const StoreConfig::Params & params); - > **Warning** - > Do not use this unless the filesystem is read-only. - > - > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. - > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. - )"}; + /** + * For `RestrictedStore` + */ + LocalStoreConfig(const LocalStoreConfig &); static const std::string name() { @@ -113,6 +76,8 @@ struct LocalStoreConfig : std::enable_shared_from_this, ref openStore() const override; StoreReference getReference() const override; + + Path getBuildDir() const; }; class LocalStore : public virtual IndirectRootStore, public virtual GcStore diff --git a/src/libstore/include/nix/store/machines.hh b/src/libstore/include/nix/store/machines.hh index 1f7bb669ab5..cea054dc142 100644 --- a/src/libstore/include/nix/store/machines.hh +++ b/src/libstore/include/nix/store/machines.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + #include "nix/util/ref.hh" #include "nix/store/store-reference.hh" diff --git a/src/libstore/include/nix/store/meson.build b/src/libstore/include/nix/store/meson.build index e41a7da4d01..b3f387947bb 100644 --- a/src/libstore/include/nix/store/meson.build +++ b/src/libstore/include/nix/store/meson.build @@ -25,6 +25,8 @@ headers = [ config_pub_h ] + files( 'common-protocol-impl.hh', 'common-protocol.hh', 'common-ssh-store-config.hh', + 'config-parse-impl.hh', + 'config-parse.hh', 'content-address.hh', 'daemon.hh', 'derivation-options.hh', @@ -32,6 +34,7 @@ headers = [ config_pub_h ] + files( 'derived-path-map.hh', 'derived-path.hh', 'downstream-placeholder.hh', + 'dummy-store.hh', 'filetransfer.hh', 'gc-store.hh', 'globals.hh', diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index 76591cf9390..fa5d7167a2f 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -17,18 +17,20 @@ struct FdSource; template class Pool; -struct RemoteStoreConfig : virtual StoreConfig +template class F> +struct RemoteStoreConfigT { - using StoreConfig::StoreConfig; + F maxConnections; + F maxConnectionAge; +}; + +struct RemoteStoreConfig : RemoteStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); - const Setting maxConnections{ - this, 1, "max-connections", "Maximum number of concurrent connections to the Nix daemon."}; + const Store::Config & storeConfig; - const Setting maxConnectionAge{ - this, - std::numeric_limits::max(), - "max-connection-age", - "Maximum age of a connection before it is closed."}; + RemoteStoreConfig(const Store::Config &, const StoreConfig::Params &); }; /** diff --git a/src/libstore/include/nix/store/s3-binary-cache-store.hh b/src/libstore/include/nix/store/s3-binary-cache-store.hh index 2fe66b0ad93..38ffc84636a 100644 --- a/src/libstore/include/nix/store/s3-binary-cache-store.hh +++ b/src/libstore/include/nix/store/s3-binary-cache-store.hh @@ -11,89 +11,33 @@ namespace nix { -struct S3BinaryCacheStoreConfig : std::enable_shared_from_this, virtual BinaryCacheStoreConfig +template class F> +struct S3BinaryCacheStoreConfigT { + F profile; + F region; + F scheme; + F endpoint; + F narinfoCompression; + F lsCompression; + F logCompression; + F multipartUpload; + F bufferSize; +}; + +struct S3BinaryCacheStoreConfig : std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig, + S3BinaryCacheStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); + + S3BinaryCacheStoreConfig( + std::string_view uriScheme, std::string_view bucketName, const StoreConfig::Params & params); + std::string bucketName; - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - - S3BinaryCacheStoreConfig(std::string_view uriScheme, std::string_view bucketName, const Params & params); - - const Setting profile{ - this, - "", - "profile", - R"( - The name of the AWS configuration profile to use. By default - Nix uses the `default` profile. - )"}; - -protected: - - constexpr static const char * defaultRegion = "us-east-1"; - -public: - - const Setting region{ - this, - defaultRegion, - "region", - R"( - The region of the S3 bucket. If your bucket is not in - `us-east-1`, you should always explicitly specify the region - parameter. - )"}; - - const Setting scheme{ - this, - "", - "scheme", - R"( - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - )"}; - - const Setting endpoint{ - this, - "", - "endpoint", - R"( - The URL of the endpoint of an S3-compatible service such as MinIO. - Do not specify this setting if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and uses path-based - > addressing instead of virtual host based addressing. - )"}; - - const Setting narinfoCompression{ - this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; - - const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; - - const Setting logCompression{ - this, - "", - "log-compression", - R"( - Compression method for `log/*` files. It is recommended to - use a compression method supported by most web browsers - (e.g. `brotli`). - )"}; - - const Setting multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."}; - - const Setting bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; - - static const std::string name() + static std::string name() { return "S3 Binary Cache Store"; } @@ -114,9 +58,9 @@ struct S3BinaryCacheStore : virtual BinaryCacheStore { using Config = S3BinaryCacheStoreConfig; - ref config; + ref config; - S3BinaryCacheStore(ref); + S3BinaryCacheStore(ref); struct Stats { diff --git a/src/libstore/include/nix/store/ssh-store.hh b/src/libstore/include/nix/store/ssh-store.hh index 9584a1a862c..3eb967cecf8 100644 --- a/src/libstore/include/nix/store/ssh-store.hh +++ b/src/libstore/include/nix/store/ssh-store.hh @@ -8,17 +8,27 @@ namespace nix { +template class F> +struct SSHStoreConfigT +{ + F remoteProgram; +}; + struct SSHStoreConfig : std::enable_shared_from_this, - virtual RemoteStoreConfig, - virtual CommonSSHStoreConfig + Store::Config, + RemoteStore::Config, + CommonSSHStoreConfig, + SSHStoreConfigT { - using CommonSSHStoreConfig::CommonSSHStoreConfig; - using RemoteStoreConfig::RemoteStoreConfig; + static config::SettingDescriptionMap descriptions(); - SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); + std::optional mounted; - const Setting remoteProgram{ - this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; + SSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreConfig::Params & params, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); static const std::string name() { @@ -37,29 +47,4 @@ struct SSHStoreConfig : std::enable_shared_from_this, StoreReference getReference() const override; }; -struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig -{ - MountedSSHStoreConfig(StringMap params); - MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params); - - static const std::string name() - { - return "Experimental SSH Store with filesystem mounted"; - } - - static StringSet uriSchemes() - { - return {"mounted-ssh-ng"}; - } - - static std::string doc(); - - static std::optional experimentalFeature() - { - return ExperimentalFeature::MountedSSHStore; - } - - ref openStore() const override; -}; - } // namespace nix diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 3e32c49a345..4c658f242ed 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -9,7 +9,6 @@ #include "nix/util/lru-cache.hh" #include "nix/util/sync.hh" #include "nix/store/globals.hh" -#include "nix/util/configuration.hh" #include "nix/store/path-info.hh" #include "nix/util/repair-flag.hh" #include "nix/store/store-dir-config.hh" @@ -25,6 +24,32 @@ namespace nix { +/** + * About the class hierarchy of the store types: + * + * Each store type `Foo` consists of two classes: + * + * 1. A class `FooConfig : virtual StoreConfig` that contains the configuration + * for the store + * + * It should only contain members of type `const Setting` (or subclasses + * of it) and inherit the constructors of `StoreConfig` + * (`using StoreConfig::StoreConfig`). + * + * 2. A class `Foo : virtual Store, virtual FooConfig` that contains the + * implementation of the store. + * + * This class is expected to have a constructor `Foo(const StoreConfig::Params & params)` + * that calls `StoreConfig(params)` (otherwise you're gonna encounter an + * `assertion failure` when trying to instantiate it). + * + * You can then register the new store using: + * + * ``` + * cpp static RegisterStoreImplementation regStore; + * ``` + */ + MakeError(SubstError, Error); /** * denotes a permanent build failure @@ -81,38 +106,40 @@ struct MissingPaths uint64_t narSize{0}; }; +template class F> +struct StoreConfigT +{ + F pathInfoCacheSize; + F isTrusted; + F systemFeatures; +}; + +template class F> +struct SubstituterConfigT +{ + F priority; + F wantMassQuery; +}; + /** - * About the class hierarchy of the store types: - * - * Each store type `Foo` consists of two classes: - * - * 1. A class `FooConfig : virtual StoreConfig` that contains the configuration - * for the store - * - * It should only contain members of type `Setting` (or subclasses - * of it) and inherit the constructors of `StoreConfig` - * (`using StoreConfig::StoreConfig`). - * - * 2. A class `Foo : virtual Store` that contains the - * implementation of the store. - * - * This class is expected to have: - * - * 1. an alias `using Config = FooConfig;` - * - * 2. a constructor `Foo(ref params)`. - * - * You can then register the new store using: - * - * ``` - * cpp static RegisterStoreImplementation regStore; - * ``` + * @note In other cases we don't expose this function directly, but in + * this case we must because of `Store::resolvedSubstConfig` below. As + * the docs of that field describe, this is a case where the + * configuration is intentionally stateful. */ -struct StoreConfig : public StoreDirConfig +SubstituterConfigT substituterConfigDefaults(); + +/** + * @note `std::optional` rather than `config::PlainValue` is applied to + * `SubstitutorConfigT` because these are overrides. Caches themselves (not our + * config) can update default settings, but aren't allowed to update settings + * specified by the client (i.e. us). + */ +struct StoreConfig : StoreDirConfig, StoreConfigT, SubstituterConfigT { - using StoreDirConfig::StoreDirConfig; + static config::SettingDescriptionMap descriptions(); - StoreConfig() = delete; + StoreConfig(const StoreConfig::Params &); virtual ~StoreConfig() {} @@ -129,14 +156,17 @@ struct StoreConfig : public StoreDirConfig /** * Get overridden store reference query parameters. */ - StringMap getQueryParams() const + StoreReference::Params getQueryParams() const { +#if 0 // FIXME render back to query parms auto queryParams = std::map{}; getSettings(queryParams, /*overriddenOnly=*/true); StringMap res; for (const auto & [name, info] : queryParams) res.insert({name, info.value}); return res; +#endif + return {}; } /** @@ -148,50 +178,6 @@ struct StoreConfig : public StoreDirConfig return std::nullopt; } - Setting pathInfoCacheSize{ - this, 65536, "path-info-cache-size", "Size of the in-memory store path metadata cache."}; - - Setting isTrusted{ - this, - false, - "trusted", - R"( - Whether paths from this store can be used as substitutes - even if they are not signed by a key listed in the - [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) - setting. - )"}; - - Setting priority{ - this, - 0, - "priority", - R"( - Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). - A lower value means a higher priority. - )"}; - - Setting wantMassQuery{ - this, - false, - "want-mass-query", - R"( - Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). - )"}; - - Setting systemFeatures{ - this, - getDefaultSystemFeatures(), - "system-features", - R"( - Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. - - Example: `"kvm"` - )", - {}, - // Don't document the machine-specific default value - false}; - /** * Open a store of the type corresponding to this configuration * type. @@ -238,6 +224,14 @@ public: return config; } + /** + * Resolved substituter configuration. This is intentionally mutable + * as store clients may do IO to ask the underlying store for their + * default setting values if the client config did not statically + * override them. + */ + SubstituterConfigT resolvedSubstConfig = substituterConfigDefaults(); + protected: struct PathInfoCacheValue @@ -281,11 +275,6 @@ protected: Store(const Store::Config & config); public: - /** - * Perform any necessary effectful operation to make the store up and - * running - */ - virtual void init() {}; virtual ~Store() {} @@ -964,3 +953,6 @@ std::map drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore = nullptr); } // namespace nix + +// Parses a Store URL, uses global state not pure so think about this +JSON_IMPL(ref) diff --git a/src/libstore/include/nix/store/store-dir-config.hh b/src/libstore/include/nix/store/store-dir-config.hh index bc2944b0b89..714a821fe2e 100644 --- a/src/libstore/include/nix/store/store-dir-config.hh +++ b/src/libstore/include/nix/store/store-dir-config.hh @@ -3,8 +3,8 @@ #include "nix/store/path.hh" #include "nix/util/hash.hh" #include "nix/store/content-address.hh" -#include "nix/store/globals.hh" -#include "nix/util/configuration.hh" +#include "nix/store/store-reference.hh" +#include "nix/store/config-parse.hh" #include #include @@ -17,6 +17,20 @@ struct SourcePath; MakeError(BadStorePath, Error); MakeError(BadStorePathName, BadStorePath); +/** + * Underlying store directory configuration type. + * + * Don't worry to much about the `F` parameter, it just some abstract + * nonsense for the "higher-kinded data" pattern. It is used in each + * settings record in order to ensure don't forgot to parse or document + * settings field. + */ +template class F> +struct StoreDirConfigT +{ + F _storeDir; +}; + /** * @todo This should just be part of `StoreDirConfig`. However, it would * be a huge amount of churn if `Store` didn't have these methods @@ -30,8 +44,6 @@ struct MixStoreDirMethods { const Path & storeDir; - // pure methods - StorePath parseStorePath(std::string_view path) const; std::optional maybeParseStorePath(std::string_view path) const; @@ -97,38 +109,25 @@ struct MixStoreDirMethods }; /** - * Need to make this a separate class so I can get the right - * initialization order in the constructor for `StoreDirConfig`. + * Store directory configuration type. + * + * Combines the underlying `*T` type (with plain values for the fields) + * and the methods. + * + * The order of `StoreDirConfigT` and then + * `MixStoreDirMethods` is very important. This ensures that + * `StoreDirConfigT::storeDir_` is initialized + * before we have our one chance (because references are immutable) to + * initialize `MixStoreDirMethods::storeDir`. */ -struct StoreDirConfigBase : Config +struct StoreDirConfig : StoreDirConfigT, MixStoreDirMethods { - using Config::Config; - - const PathSetting storeDir_{ - this, - settings.nixStore, - "store", - R"( - Logical location of the Nix store, usually - `/nix/store`. Note that you can only copy store paths - between stores if they have the same `store` setting. - )"}; -}; + using Params = StoreReference::Params; -/** - * The order of `StoreDirConfigBase` and then `MixStoreDirMethods` is - * very important. This ensures that `StoreDirConfigBase::storeDir_` - * is initialized before we have our one chance (because references are - * immutable) to initialize `MixStoreDirMethods::storeDir`. - */ -struct StoreDirConfig : StoreDirConfigBase, MixStoreDirMethods -{ - using Params = StringMap; + static config::SettingDescriptionMap descriptions(); StoreDirConfig(const Params & params); - StoreDirConfig() = delete; - virtual ~StoreDirConfig() = default; }; diff --git a/src/libstore/include/nix/store/store-reference.hh b/src/libstore/include/nix/store/store-reference.hh index fff3b5c5cd4..ae5f0e001e4 100644 --- a/src/libstore/include/nix/store/store-reference.hh +++ b/src/libstore/include/nix/store/store-reference.hh @@ -2,8 +2,10 @@ ///@file #include +#include #include "nix/util/types.hh" +#include "nix/util/json-impls.hh" namespace nix { @@ -41,7 +43,17 @@ namespace nix { */ struct StoreReference { - using Params = StringMap; + /** + * Would do + * + * ``` + * using Params = nlohmann::json::object_t; + * ``` + * + * but cannot because `` doesn't have that. + * + */ + using Params = std::map>; /** * Special store reference `""` or `"auto"` @@ -70,7 +82,7 @@ struct StoreReference Params params; - bool operator==(const StoreReference & rhs) const = default; + bool operator==(const StoreReference & rhs) const; /** * Render the whole store reference as a URI, including parameters. @@ -89,3 +101,5 @@ struct StoreReference std::pair splitUriAndParams(const std::string & uri); } // namespace nix + +JSON_IMPL(StoreReference) diff --git a/src/libstore/include/nix/store/store-registration.hh b/src/libstore/include/nix/store/store-registration.hh index 8b0f344ba38..e48d5fd40af 100644 --- a/src/libstore/include/nix/store/store-registration.hh +++ b/src/libstore/include/nix/store/store-registration.hh @@ -29,6 +29,19 @@ struct StoreFactory */ StringSet uriSchemes; + /** + * @note This is a functional pointer for now because this situation: + * + * - We register store types with global initializers + * + * - The default values for some settings maybe depend on the settings globals. + * + * And because the ordering of global initialization is arbitrary, + * this is not allowed. For now, we can simply defer actually + * creating these maps until we need to later. + */ + config::SettingDescriptionMap (*configDescriptions)(); + /** * An experimental feature this type store is gated, if it is to be * experimental. @@ -40,21 +53,21 @@ struct StoreFactory * whatever comes after `://` and before `?`. */ std::function( - std::string_view scheme, std::string_view authorityPath, const Store::Config::Params & params)> + std::string_view scheme, std::string_view authorityPath, const StoreConfig::Params & params)> parseConfig; - - /** - * Just for dumping the defaults. Kind of awkward this exists, - * because it means we cannot require fields to be manually - * specified so easily. - */ - std::function()> getConfig; }; struct Implementations { +private: + + /** + * The name of this type of store, and a factory for it. + */ using Map = std::map; +public: + static Map & registered(); template @@ -63,11 +76,11 @@ struct Implementations StoreFactory factory{ .doc = TConfig::doc(), .uriSchemes = TConfig::uriSchemes(), + .configDescriptions = TConfig::descriptions, .experimentalFeature = TConfig::experimentalFeature(), .parseConfig = ([](auto scheme, auto uri, auto & params) -> ref { return make_ref(scheme, uri, params); }), - .getConfig = ([]() -> ref { return make_ref(Store::Config::Params{}); }), }; auto [it, didInsert] = registered().insert({TConfig::name(), std::move(factory)}); if (!didInsert) { diff --git a/src/libstore/include/nix/store/uds-remote-store.hh b/src/libstore/include/nix/store/uds-remote-store.hh index 37c239796d9..435871bed48 100644 --- a/src/libstore/include/nix/store/uds-remote-store.hh +++ b/src/libstore/include/nix/store/uds-remote-store.hh @@ -8,20 +8,21 @@ namespace nix { struct UDSRemoteStoreConfig : std::enable_shared_from_this, - virtual LocalFSStoreConfig, - virtual RemoteStoreConfig + Store::Config, + LocalFSStore::Config, + RemoteStore::Config { - // TODO(fzakaria): Delete this constructor once moved over to the factory pattern - // outlined in https://github.com/NixOS/nix/issues/10766 - using LocalFSStoreConfig::LocalFSStoreConfig; - using RemoteStoreConfig::RemoteStoreConfig; + static config::SettingDescriptionMap descriptions(); + + UDSRemoteStoreConfig(const StoreConfig::Params & params) + : UDSRemoteStoreConfig{"unix", "", params} + { + } /** * @param authority is the socket path. */ - UDSRemoteStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); - - UDSRemoteStoreConfig(const Params & params); + UDSRemoteStoreConfig(std::string_view scheme, std::string_view authority, const StoreConfig::Params & params); static const std::string name() { diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 0435cfa6249..3b2ceebf7f1 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -12,14 +12,61 @@ #include "nix/store/ssh.hh" #include "nix/store/derivations.hh" #include "nix/util/callback.hh" +#include "nix/store/config-parse-impl.hh" #include "nix/store/store-registration.hh" namespace nix { -LegacySSHStoreConfig::LegacySSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - , CommonSSHStoreConfig(scheme, ParsedURL::Authority::parse(authority), params) +constexpr static const LegacySSHStoreConfigT legacySSHStoreConfigDescriptions = { + .remoteProgram{ + .name = "remote-program", + .description = "Path to the `nix-store` executable on the remote machine.", + }, + .maxConnections{ + .name = "max-connections", + .description = "Maximum number of concurrent SSH connections.", + }, +}; + +#define LEGACY_SSH_STORE_CONFIG_FIELDS(X) X(remoteProgram), X(maxConnections) + +MAKE_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS) + +static LegacySSHStoreConfigT legacySSHStoreConfigDefaults() { + return { + .remoteProgram = {{"nix-store"}}, + .maxConnections = {1}, + }; +} + +MAKE_APPLY_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap LegacySSHStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(CommonSSHStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + { + constexpr auto & descriptions = legacySSHStoreConfigDescriptions; + auto defaults = legacySSHStoreConfigDefaults(); + ret.merge(decltype(ret){LEGACY_SSH_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}); + } + return ret; +} + +LegacySSHStore::Config::LegacySSHStoreConfig( + std::string_view scheme, std::string_view authority, const StoreConfig::Params & params) + : Store::Config{params} + , CommonSSHStoreConfig{scheme, ParsedURL::Authority::parse(authority), params} + , LegacySSHStoreConfigT{legacySSHStoreConfigApplyParse(params)} +{ +#ifndef _WIN32 + if (auto * p = get(params, "log-fd")) { + logFD = p->get(); + } +#endif } std::string LegacySSHStoreConfig::doc() diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index b5e43de68b4..6851873f9b0 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -8,10 +8,18 @@ namespace nix { +config::SettingDescriptionMap LocalBinaryCacheStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(BinaryCacheStoreConfig::descriptions()); + return ret; +} + LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( std::string_view scheme, PathView binaryCacheDir, const StoreReference::Params & params) : Store::Config{params} - , BinaryCacheStoreConfig{params} + , BinaryCacheStoreConfig{*this, params} , binaryCacheDir(binaryCacheDir) { } @@ -38,9 +46,9 @@ struct LocalBinaryCacheStore : virtual BinaryCacheStore { using Config = LocalBinaryCacheStoreConfig; - ref config; + ref config; - LocalBinaryCacheStore(ref config) + LocalBinaryCacheStore(ref config) : Store{*config} , BinaryCacheStore{*config} , config{config} @@ -125,9 +133,7 @@ StringSet LocalBinaryCacheStoreConfig::uriSchemes() ref LocalBinaryCacheStoreConfig::openStore() const { - auto store = make_ref( - ref{// FIXME we shouldn't actually need a mutable config - std::const_pointer_cast(shared_from_this())}); + auto store = make_ref(ref{shared_from_this()}); store->init(); return store; } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index fd1fe44592b..69211ca2297 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,3 +1,4 @@ +#include "nix/util/json-utils.hh" #include "nix/util/archive.hh" #include "nix/util/posix-source-accessor.hh" #include "nix/store/store-api.hh" @@ -5,18 +6,83 @@ #include "nix/store/globals.hh" #include "nix/util/compression.hh" #include "nix/store/derivations.hh" +#include "nix/store/config-parse-impl.hh" namespace nix { -LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params) - : StoreConfig(params) - // Default `?root` from `rootDir` if non set - // FIXME don't duplicate description once we don't have root setting - , rootDir{ - this, - !rootDir.empty() && params.count("root") == 0 ? (std::optional{rootDir}) : std::nullopt, - "root", - "Directory prefixed to all other paths."} +constexpr static const LocalFSStoreConfigT localFSStoreConfigDescriptions = { + .rootDir = + { + .name = "root", + .description = "Directory prefixed to all other paths.", + }, + .stateDir = + { + .name = "state", + .description = "Directory where Nix stores state.", + }, + .logDir = + { + .name = "log", + .description = "directory where Nix stores log files.", + }, + .realStoreDir{ + .name = "real", + .description = "Physical path of the Nix store.", + }, +}; + +#define LOCAL_FS_STORE_CONFIG_FIELDS(X) X(rootDir), X(stateDir), X(logDir), X(realStoreDir), + +MAKE_PARSE(LocalFSStoreConfig, localFSStoreConfig, LOCAL_FS_STORE_CONFIG_FIELDS) + +/** + * @param rootDir Fallback if not in `params` + */ +static LocalFSStoreConfigT +localFSStoreConfigDefaults(const Path & storeDir, const std::optional & rootDir) +{ + return { + .rootDir = {std::nullopt}, + .stateDir = {rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir}, + .logDir = {rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir}, + .realStoreDir = {rootDir ? *rootDir + "/nix/store" : storeDir}, + }; +} + +static LocalFSStoreConfigT +localFSStoreConfigApplyParse(const Path & storeDir, LocalFSStoreConfigT parsed) +{ + auto defaults = localFSStoreConfigDefaults(storeDir, parsed.rootDir.value_or(std::nullopt)); + return {LOCAL_FS_STORE_CONFIG_FIELDS(APPLY_ROW)}; +} + +config::SettingDescriptionMap LocalFSStoreConfig::descriptions() +{ + constexpr auto & descriptions = localFSStoreConfigDescriptions; + auto defaults = localFSStoreConfigDefaults(settings.nixStore, std::nullopt); + return {LOCAL_FS_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}; +} + +LocalFSStore::Config::LocalFSStoreConfig(const Store::Config & storeConfig, const StoreConfig::Params & params) + : LocalFSStoreConfigT{localFSStoreConfigApplyParse( + storeConfig.storeDir, localFSStoreConfigParse(params))} + , storeConfig{storeConfig} +{ +} + +static LocalFSStoreConfigT applyAuthority(LocalFSStoreConfigT parsed, PathView rootDir) +{ + if (!rootDir.empty()) + parsed.rootDir = std::optional{Path{rootDir}}; + return parsed; +} + +LocalFSStore::Config::LocalFSStoreConfig( + const Store::Config & storeConfig, PathView rootDir, const StoreConfig::Params & params) + : LocalFSStoreConfigT{localFSStoreConfigApplyParse( + storeConfig.storeDir, applyAuthority(localFSStoreConfigParse(params), rootDir))} + , storeConfig{storeConfig} { } diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 2b000b3dba6..e7865feb9fc 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -5,11 +5,97 @@ #include "nix/store/realisation.hh" #include "nix/util/processes.hh" #include "nix/util/url.hh" -#include "nix/store/store-open.hh" +#include "nix/store/store-api.hh" #include "nix/store/store-registration.hh" +#include "nix/store/config-parse-impl.hh" namespace nix { +static LocalOverlayStoreConfigT localOverlayStoreConfigDescriptions = { + .lowerStoreConfig{ + .name = "lower-store", + .description = R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). + + Must be a store with a store dir on the file system. + Must be used as OverlayFS lower layer for this store's store dir. + )", + // It's not actually machine-specific, but we don't yet have a + // `to_json` for `StoreConfig`. + .documentDefault = false, + }, + .upperLayer{ + .name = "upper-layer", + .description = R"( + Directory containing the OverlayFS upper layer for this store's store dir. + )", + }, + .checkMount{ + .name = "check-mount", + .description = R"( + Check that the overlay filesystem is correctly mounted. + + Nix does not manage the overlayfs mount point itself, but the correct + functioning of the overlay store does depend on this mount point being set up + correctly. Rather than just assume this is the case, check that the lowerdir + and upperdir options are what we expect them to be. This check is on by + default, but can be disabled if needed. + )", + }, + .remountHook{ + .name = "remount-hook", + .description = R"( + Script or other executable to run when overlay filesystem needs remounting. + + This is occasionally necessary when deleting a store path that exists in both upper and lower layers. + In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly + is the only way to perform the deletion without creating a "whiteout". + However this causes the OverlayFS kernel data structures to get out-of-sync, + and can lead to 'stale file handle' errors; remounting solves the problem. + + The store directory is passed as an argument to the invoked executable. + )", + }, +}; + +#define LOCAL_OVERLAY_STORE_CONFIG_FIELDS(X) X(lowerStoreConfig), X(upperLayer), X(checkMount), X(remountHook), + +MAKE_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS) + +static LocalOverlayStoreConfigT localOverlayStoreConfigDefaults() +{ + return { + .lowerStoreConfig = {make_ref(StoreConfig::Params{})}, + .upperLayer = {""}, + .checkMount = {true}, + .remountHook = {""}, + }; +} + +MAKE_APPLY_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap LocalOverlayStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + ret.merge(LocalStoreConfig::descriptions()); + { + constexpr auto & descriptions = localOverlayStoreConfigDescriptions; + auto defaults = localOverlayStoreConfigDefaults(); + ret.merge(decltype(ret){LOCAL_OVERLAY_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}); + } + return ret; +} + +LocalOverlayStore::Config::LocalOverlayStoreConfig( + std::string_view scheme, std::string_view authority, const StoreConfig::Params & params) + : LocalStore::Config(scheme, authority, params) + , LocalOverlayStoreConfigT{localOverlayStoreConfigApplyParse(params)} +{ +} + std::string LocalOverlayStoreConfig::doc() { return @@ -43,7 +129,7 @@ LocalOverlayStore::LocalOverlayStore(ref config) , LocalFSStore{*config} , LocalStore{static_cast>(config)} , config{config} - , lowerStore(openStore(percentDecode(config->lowerStoreUri.get())).dynamic_pointer_cast()) + , lowerStore(config->lowerStoreConfig.value->openStore().dynamic_pointer_cast()) { if (config->checkMount.get()) { std::smatch match; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f4d1b66ba93..f6eefe81884 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -18,6 +18,7 @@ #include "nix/store/keys.hh" #include "nix/util/url.hh" #include "nix/util/users.hh" +#include "nix/store/config-parse-impl.hh" #include "nix/store/store-open.hh" #include "nix/store/store-registration.hh" @@ -56,17 +57,99 @@ #include #include "nix/util/strings.hh" +#include "nix/util/json-utils.hh" #include "store-config-private.hh" namespace nix { -LocalStoreConfig::LocalStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(authority, params) +constexpr static const LocalStoreConfigT localStoreConfigDescriptions = { + .requireSigs = + { + .name = "require-sigs", + .description = "Whether store paths copied into this store should have a trusted signature.", + }, + .readOnly = + { + .name = "read-only", + .description = R"( + Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. + + Normally Nix attempts to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. + + Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. + + > **Warning** + > Do not use this unless the filesystem is read-only. + > + > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. + > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. + )", + }, + .buildDir = + { + .name = "build-dir", + .description = R"( + The directory on the host, in which derivations' temporary build directories are created. + + If not set, Nix will use the `builds` subdirectory of its configured state directory. + + Note that builds are often performed by the Nix daemon, so its `build-dir` applies. + + Nix will create this directory automatically with suitable permissions if it does not exist. + Otherwise its permissions must allow all users to traverse the directory (i.e. it must have `o+x` set, in unix parlance) for non-sandboxed builds to work correctly. + + This is also the location where [`--keep-failed`](@docroot@/command-ref/opt-common.md#opt-keep-failed) leaves its files. + + If Nix runs without sandbox, or if the platform does not support sandboxing with bind mounts (e.g. macOS), then the [`builder`](@docroot@/language/derivations.md#attr-builder)'s environment will contain this directory, instead of the virtual location [`sandbox-build-dir`](#conf-sandbox-build-dir). + + > **Warning** + > + > `build-dir` must not be set to a world-writable directory. + > Placing temporary build directories in a world-writable place allows other users to access or modify build data that is currently in use. + > This alone is merely an impurity, but combined with another factor this has allowed malicious derivations to escape the build sandbox. + )", + }, +}; + +#define LOCAL_STORE_CONFIG_FIELDS(X) X(requireSigs), X(readOnly), X(buildDir), + +MAKE_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS) + +static LocalStoreConfigT localStoreConfigDefaults() +{ + return { + .requireSigs = {settings.requireSigs}, + .readOnly = {false}, + .buildDir = {std::nullopt}, + }; +} + +MAKE_APPLY_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap LocalStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + { + constexpr auto & descriptions = localStoreConfigDescriptions; + auto defaults = localStoreConfigDefaults(); + ret.merge(decltype(ret){LOCAL_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}); + } + return ret; +} + +LocalStore::Config::LocalStoreConfig( + std::string_view scheme, std::string_view authority, const StoreConfig::Params & params) + : Store::Config(params) + , LocalFSStore::Config(*this, authority, params) + , LocalStoreConfigT{localStoreConfigApplyParse(params)} { } +LocalStoreConfig::LocalStoreConfig(const LocalStoreConfig &) = default; + std::string LocalStoreConfig::doc() { return @@ -74,7 +157,7 @@ std::string LocalStoreConfig::doc() ; } -Path LocalBuildStoreConfig::getBuildDir() const +Path LocalStoreConfig::getBuildDir() const { return settings.buildDir.get().has_value() ? *settings.buildDir.get() : buildDir.get().has_value() ? *buildDir.get() @@ -912,7 +995,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) break; if (sub->storeDir != storeDir) continue; - if (!sub->config.wantMassQuery) + if (!sub->resolvedSubstConfig.wantMassQuery) continue; auto valid = sub->queryValidPaths(remaining); diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index d614676668b..79496a2cbcd 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -64,8 +64,8 @@ StoreReference Machine::completeStoreReference() const auto * generic = std::get_if(&storeUri.variant); if (generic && generic->scheme == "ssh") { - storeUri.params["max-connections"] = "1"; - storeUri.params["log-fd"] = "4"; + storeUri.params["max-connections"] = 1; + storeUri.params["log-fd"] = 4; } if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) { @@ -77,15 +77,12 @@ StoreReference Machine::completeStoreReference() const { auto & fs = storeUri.params["system-features"]; - auto append = [&](auto feats) { - for (auto & f : feats) { - if (fs.size() > 0) - fs += ' '; - fs += f; - } - }; - append(supportedFeatures); - append(mandatoryFeatures); + if (!fs.is_array()) + fs = nlohmann::json::array(); + auto features = supportedFeatures; + features.insert(supportedFeatures.begin(), supportedFeatures.end()); + for (auto & feat : features) + fs += feat; } return storeUri; diff --git a/src/libstore/meson.build b/src/libstore/meson.build index ad76582d8e3..81d4bc1259c 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -277,6 +277,7 @@ sources = files( 'builtins/unpack-channel.cc', 'common-protocol.cc', 'common-ssh-store-config.cc', + 'config-parse.cc', 'content-address.cc', 'daemon.cc', 'derivation-options.cc', diff --git a/src/libstore/mounted-ssh-store.md b/src/libstore/mounted-ssh-store.md deleted file mode 100644 index 1ebfe3081dc..00000000000 --- a/src/libstore/mounted-ssh-store.md +++ /dev/null @@ -1,18 +0,0 @@ -R"( - -**Store URL format**: `mounted-ssh-ng://[username@]hostname` - -Experimental store type that allows full access to a Nix store on a remote machine, -and additionally requires that store be mounted in the local file system. - -The mounting of that store is not managed by Nix, and must by managed manually. -It could be accomplished with SSHFS or NFS, for example. - -The local file system is used to optimize certain operations. -For example, rather than serializing Nix archives and sending over the Nix channel, -we can directly access the file system data via the mount-point. - -The local file system is also used to make certain operations possible that wouldn't otherwise be. -For example, persistent GC roots can be created if they reside on the same file system as the remote store: -the remote side will create the symlinks necessary to avoid race conditions. -)" diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 3eff339e114..b4c415adce8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -18,14 +18,53 @@ #include "nix/util/callback.hh" #include "nix/store/filetransfer.hh" #include "nix/util/signals.hh" +#include "nix/store/config-parse-impl.hh" #include namespace nix { +constexpr static const RemoteStoreConfigT remoteStoreConfigDescriptions = { + .maxConnections{ + .name = "max-connections", + .description = "Maximum number of concurrent connections to the Nix daemon.", + }, + .maxConnectionAge{ + .name = "max-connection-age", + .description = "Maximum age of a connection before it is closed.", + }, +}; + +#define REMOTE_STORE_CONFIG_FIELDS(X) X(maxConnections), X(maxConnectionAge), + +MAKE_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS) + +static RemoteStoreConfigT remoteStoreConfigDefaults() +{ + return { + .maxConnections = {1}, + .maxConnectionAge = {std::numeric_limits::max()}, + }; +} + +MAKE_APPLY_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap RemoteStoreConfig::descriptions() +{ + constexpr auto & descriptions = remoteStoreConfigDescriptions; + auto defaults = remoteStoreConfigDefaults(); + return {REMOTE_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}; +} + +RemoteStore::Config::RemoteStoreConfig(const Store::Config & storeConfig, const StoreConfig::Params & params) + : RemoteStoreConfigT{remoteStoreConfigApplyParse(params)} + , storeConfig{storeConfig} +{ +} + /* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Config & config) - : Store{config} + : Store{config.storeConfig} , config{config} , connections( make_ref>( @@ -53,7 +92,7 @@ RemoteStore::RemoteStore(const Config & config) ref RemoteStore::openConnectionWrapper() { if (failed) - throw Error("opening a connection to remote store '%s' previously failed", config.getUri()); + throw Error("opening a connection to remote store '%s' previously failed", config.storeConfig.getUri()); try { return openConnection(); } catch (...) { @@ -95,7 +134,7 @@ void RemoteStore::initConnection(Connection & conn) if (ex) std::rethrow_exception(ex); } catch (Error & e) { - throw Error("cannot open connection to remote store '%s': %s", config.getUri(), e.what()); + throw Error("cannot open connection to remote store '%s': %s", config.storeConfig.getUri(), e.what()); } setOptions(conn); diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 84eb63f7fc3..b1108d99c3f 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -2,8 +2,6 @@ #if NIX_WITH_S3_SUPPORT -# include - # include "nix/store/s3.hh" # include "nix/store/nar-info.hh" # include "nix/store/nar-info-disk-cache.hh" @@ -11,6 +9,7 @@ # include "nix/util/compression.hh" # include "nix/store/filetransfer.hh" # include "nix/util/signals.hh" +# include "nix/store/config-parse-impl.hh" # include "nix/store/store-registration.hh" # include @@ -226,22 +225,121 @@ S3Helper::FileTransferResult S3Helper::getObject(const std::string & bucketName, return res; } -S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( - std::string_view uriScheme, std::string_view bucketName, const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , bucketName(bucketName) +constexpr static const S3BinaryCacheStoreConfigT s3BinaryCacheStoreConfigDescriptions = { + .profile{ + .name = "profile", + .description = R"( + The name of the AWS configuration profile to use. By default + Nix uses the `default` profile. + )", + }, + .region{ + .name = "region", + .description = R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )", + }, + .scheme{ + .name = "scheme", + .description = R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )", + }, + .endpoint{ + .name = "endpoint", + .description = R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and uses path-based + > addressing instead of virtual host based addressing. + )", + }, + .narinfoCompression{ + .name = "narinfo-compression", + .description = "Compression method for `.narinfo` files.", + }, + .lsCompression{ + .name = "ls-compression", + .description = "Compression method for `.ls` files.", + }, + .logCompression{ + .name = "log-compression", + .description = R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )", + }, + .multipartUpload{ + .name = "multipart-upload", + .description = "Whether to use multi-part uploads.", + }, + .bufferSize{ + .name = "buffer-size", + .description = "Size (in bytes) of each part in multi-part uploads.", + }, +}; + +# define S3_BINARY_CACHE_STORE_CONFIG_FIELDS(X) \ + X(profile), X(region), X(scheme), X(endpoint), X(narinfoCompression), X(lsCompression), X(logCompression), \ + X(multipartUpload), X(bufferSize), + +MAKE_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS) + +static S3BinaryCacheStoreConfigT s3BinaryCacheStoreConfigDefaults() { - // Don't want to use use AWS SDK in header, so we check the default - // here. TODO do this better after we overhaul the store settings - // system. - assert(std::string{defaultRegion} == std::string{Aws::Region::US_EAST_1}); + return { + .profile = {""}, + .region = {Aws::Region::US_EAST_1}, + .scheme = {""}, + .endpoint = {""}, + .narinfoCompression = {""}, + .lsCompression = {""}, + .logCompression = {""}, + .multipartUpload = {false}, + .bufferSize = {5 * 1024 * 1024}, + }; +} +MAKE_APPLY_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap S3BinaryCacheStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(BinaryCacheStoreConfig::descriptions()); + { + constexpr auto & descriptions = s3BinaryCacheStoreConfigDescriptions; + auto defaults = s3BinaryCacheStoreConfigDefaults(); + ret.merge(decltype(ret){S3_BINARY_CACHE_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}); + } + return ret; +} + +S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( + std::string_view scheme, std::string_view authority, const StoreConfig::Params & params) + : Store::Config{params} + , BinaryCacheStore::Config{*this, params} + , S3BinaryCacheStoreConfigT{s3BinaryCacheStoreConfigApplyParse(params)} + , bucketName{authority} +{ if (bucketName.empty()) - throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); + throw UsageError("`%s` store requires a bucket name in its Store URI", scheme); } -S3BinaryCacheStore::S3BinaryCacheStore(ref config) +S3BinaryCacheStore::S3BinaryCacheStore(ref config) : BinaryCacheStore(*config) , config{config} { @@ -271,7 +369,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore S3Helper s3Helper; - S3BinaryCacheStoreImpl(ref config) + S3BinaryCacheStoreImpl(ref config) : Store{*config} , BinaryCacheStore{*config} , S3BinaryCacheStore{config} @@ -283,11 +381,13 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore void init() override { if (auto cacheInfo = diskCache->upToDateCacheExists(config->getUri())) { - config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); - config->priority.setDefault(cacheInfo->priority); + resolvedSubstConfig.wantMassQuery.value = + config->storeConfig.wantMassQuery.value_or(cacheInfo->wantMassQuery); + resolvedSubstConfig.priority.value = config->storeConfig.priority.value_or(cacheInfo->priority); } else { BinaryCacheStore::init(); - diskCache->createCache(config->getUri(), config->storeDir, config->wantMassQuery, config->priority); + diskCache->createCache( + config->getUri(), config->storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority); } } @@ -573,9 +673,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore ref S3BinaryCacheStoreImpl::Config::openStore() const { - auto store = - make_ref(ref{// FIXME we shouldn't actually need a mutable config - std::const_pointer_cast(shared_from_this())}); + auto store = make_ref(ref{shared_from_this()}); store->init(); return store; } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index dafe14fea76..b8d3e190540 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,3 +1,4 @@ +#include "nix/util/json-utils.hh" #include "nix/store/ssh-store.hh" #include "nix/store/local-fs-store.hh" #include "nix/store/remote-store-connection.hh" @@ -7,14 +8,91 @@ #include "nix/store/worker-protocol-impl.hh" #include "nix/util/pool.hh" #include "nix/store/ssh.hh" +#include "nix/store/config-parse-impl.hh" #include "nix/store/store-registration.hh" namespace nix { -SSHStoreConfig::SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) +constexpr static const SSHStoreConfigT sshStoreConfigDescriptions = { + .remoteProgram{ + .name = "remote-program", + .description = "Path to the `nix-daemon` executable on the remote machine.", + }, +}; + +#define SSH_STORE_CONFIG_FIELDS(X) X(remoteProgram) + +MAKE_PARSE(SSHStoreConfig, sshStoreConfig, SSH_STORE_CONFIG_FIELDS) + +static SSHStoreConfigT sshStoreConfigDefaults() +{ + return { + .remoteProgram = {{"nix-daemon"}}, + }; +} + +MAKE_APPLY_PARSE(SSHStoreConfig, sshStoreConfig, SSH_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap SSHStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(CommonSSHStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + { + constexpr auto & descriptions = sshStoreConfigDescriptions; + auto defaults = sshStoreConfigDefaults(); + ret.merge(decltype(ret){SSH_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}); + } + ret.insert_or_assign( + "mounted", + config::SettingDescription{ + .description = stripIndentation(R"( + If this nested settings object is defined (`{..}` not `null`), additionally requires that store be mounted in the local file system. + + The mounting of that store is not managed by Nix, and must by managed manually. + It could be accomplished with SSHFS or NFS, for example. + + The local file system is used to optimize certain operations. + For example, rather than serializing Nix archives and sending over the Nix channel, + we can directly access the file system data via the mount-point. + + The local file system is also used to make certain operations possible that wouldn't otherwise be. + For example, persistent GC roots can be created if they reside on the same file system as the remote store: + the remote side will create the symlinks necessary to avoid race conditions. + )"), + .experimentalFeature = Xp::MountedSSHStore, + .info = config::SettingDescription::Sub{.nullable = true, .map = LocalFSStoreConfig::descriptions()}, + }); + return ret; +} + +static std::optional getMounted( + const Store::Config & storeConfig, + const StoreConfig::Params & params, + const ExperimentalFeatureSettings & xpSettings) +{ + auto mountedParamsOpt = optionalValueAt(params, "mounted"); + if (!mountedParamsOpt) + return {}; + auto * mountedParamsP = getNullable(*mountedParamsOpt); + xpSettings.require(Xp::MountedSSHStore); + if (!mountedParamsP) + return {}; + auto & mountedParams = getObject(*mountedParamsP); + return {{storeConfig, mountedParams}}; +} + +SSHStoreConfig::SSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreConfig::Params & params, + const ExperimentalFeatureSettings & xpSettings) : Store::Config{params} - , RemoteStore::Config{params} + , RemoteStore::Config{*this, params} , CommonSSHStoreConfig{scheme, authority, params} + , SSHStoreConfigT{sshStoreConfigApplyParse(params)} + , mounted{getMounted(*this, params, xpSettings)} { } @@ -87,31 +165,6 @@ struct SSHStore : virtual RemoteStore }; }; -MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) -{ -} - -MountedSSHStoreConfig::MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(scheme, host, params) - , LocalFSStoreConfig(params) -{ -} - -std::string MountedSSHStoreConfig::doc() -{ - return -#include "mounted-ssh-store.md" - ; -} - /** * The mounted ssh store assumes that filesystems on the remote host are * shared with the local host. This means that the remote nix store is @@ -128,13 +181,16 @@ std::string MountedSSHStoreConfig::doc() */ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore { - using Config = MountedSSHStoreConfig; + using Config = SSHStore::Config; + + const LocalFSStore::Config & mountedConfig; - MountedSSHStore(ref config) + MountedSSHStore(ref config, const LocalFSStore::Config & mountedConfig) : Store{*config} , RemoteStore{*config} , SSHStore{config} - , LocalFSStore{*config} + , LocalFSStore{mountedConfig} + , mountedConfig{mountedConfig} { extraRemoteProgramArgs = { "--process-ops", @@ -182,14 +238,14 @@ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore } }; -ref SSHStore::Config::openStore() const -{ - return make_ref(ref{shared_from_this()}); -} - ref MountedSSHStore::Config::openStore() const { - return make_ref(ref{std::dynamic_pointer_cast(shared_from_this())}); + ref config{shared_from_this()}; + + if (config->mounted) + return make_ref(config, *config->mounted); + else + return make_ref(config); } ref SSHStore::openConnection() @@ -209,6 +265,5 @@ ref SSHStore::openConnection() } static RegisterStoreImplementation regSSHStore; -static RegisterStoreImplementation regMountedSSHStore; } // namespace nix diff --git a/src/libstore/ssh-store.md b/src/libstore/ssh-store.md index 26e0d6e39e0..63a455966ee 100644 --- a/src/libstore/ssh-store.md +++ b/src/libstore/ssh-store.md @@ -4,5 +4,4 @@ R"( Experimental store type that allows full access to a Nix store on a remote machine. - )" diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index a30a079529c..6c75be0f3b8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -17,6 +17,7 @@ // `addMultipleToStore`. #include "nix/store/worker-protocol.hh" #include "nix/util/signals.hh" +#include "nix/store/config-parse-impl.hh" #include #include @@ -62,6 +63,98 @@ StorePath Store::followLinksToStorePath(std::string_view path) const return toStorePath(followLinksToStore(path)).first; } +constexpr static const StoreConfigT storeConfigDescriptions = { + .pathInfoCacheSize{ + .name = "path-info-cache-size", + .description = "Size of the in-memory store path metadata cache.", + }, + .isTrusted{ + .name = "trusted", + .description = R"( + Whether paths from this store can be used as substitutes + even if they are not signed by a key listed in the + [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) + setting. + )", + }, + .systemFeatures{ + .name = "system-features", + .description = R"( + Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. + + Example: `"kvm"` + )", + // The default value is CPU- and OS-specific, and thus + // unsuitable to be rendered in the documentation. + .documentDefault = false, + }, +}; + +constexpr static const SubstituterConfigT substituterConfigDescriptions = { + .priority{ + .name = "priority", + .description = R"( + Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + A lower value means a higher priority. + )", + }, + .wantMassQuery{ + .name = "want-mass-query", + .description = R"( + Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + )", + }, +}; + +#define STORE_CONFIG_FIELDS(X) X(pathInfoCacheSize), X(isTrusted), X(systemFeatures), + +#define SUBSTITUTER_CONFIG_FIELDS(X) X(priority), X(wantMassQuery), + +MAKE_PARSE(StoreConfig, storeConfig, STORE_CONFIG_FIELDS) +MAKE_PARSE(SubstituterConfig, substituterConfig, SUBSTITUTER_CONFIG_FIELDS) + +static StoreConfigT storeConfigDefaults() +{ + return { + .pathInfoCacheSize = {65536}, + .isTrusted = {false}, + .systemFeatures = {StoreConfig::getDefaultSystemFeatures()}, + }; +}; + +SubstituterConfigT substituterConfigDefaults() +{ + return { + .priority = {0}, + .wantMassQuery = {false}, + }; +}; + +MAKE_APPLY_PARSE(StoreConfig, storeConfig, STORE_CONFIG_FIELDS) + +Store::Config::StoreConfig(const StoreConfig::Params & params) + : StoreDirConfig{params} + , StoreConfigT{storeConfigApplyParse(params)} + , SubstituterConfigT{substituterConfigParse(params)} +{ +} + +config::SettingDescriptionMap StoreConfig::descriptions() +{ + auto ret = StoreDirConfig::descriptions(); + { + constexpr auto & descriptions = storeConfigDescriptions; + auto defaults = storeConfigDefaults(); + ret.merge(config::SettingDescriptionMap{STORE_CONFIG_FIELDS(DESCRIBE_ROW)}); + } + { + constexpr auto & descriptions = substituterConfigDescriptions; + auto defaults = substituterConfigDefaults(); + ret.merge(config::SettingDescriptionMap{SUBSTITUTER_CONFIG_FIELDS(DESCRIBE_ROW)}); + }; + return ret; +} + StorePath Store::addToStore( std::string_view name, const SourcePath & path, diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index 62f08d819c5..3fe108d8908 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -1,4 +1,5 @@ #include "nix/util/source-path.hh" +#include "nix/store/config-parse-impl.hh" #include "nix/util/util.hh" #include "nix/store/store-dir-config.hh" #include "nix/store/derivations.hh" @@ -173,10 +174,41 @@ std::pair MixStoreDirMethods::computeStorePath( }; } -StoreDirConfig::StoreDirConfig(const Params & params) - : StoreDirConfigBase(params) - , MixStoreDirMethods{storeDir_} +constexpr static const StoreDirConfigT storeDirConfigDescriptions = { + ._storeDir{ + .name = "store", + .description = R"( + Logical location of the Nix store, usually + `/nix/store`. Note that you can only copy store paths + between stores if they have the same `store` setting. + )", + }, +}; + +#define STORE_DIR_CONFIG_FIELDS(X) X(_storeDir), + +MAKE_PARSE(StoreDirConfig, storeDirConfig, STORE_DIR_CONFIG_FIELDS) + +static StoreDirConfigT storeDirConfigDefaults() +{ + return { + ._storeDir = {settings.nixStore}, + }; +} + +MAKE_APPLY_PARSE(StoreDirConfig, storeDirConfig, STORE_DIR_CONFIG_FIELDS) + +StoreDirConfig::StoreDirConfig(const StoreDirConfig::Params & params) + : StoreDirConfigT{storeDirConfigApplyParse(params)} + , MixStoreDirMethods{_storeDir.value} +{ +} + +config::SettingDescriptionMap StoreDirConfig::descriptions() { + constexpr auto & descriptions = storeDirConfigDescriptions; + auto defaults = storeDirConfigDefaults(); + return {STORE_DIR_CONFIG_FIELDS(DESCRIBE_ROW)}; } } // namespace nix diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index 13feeae3ebe..f93ba0204e4 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -1,13 +1,18 @@ #include +#include + #include "nix/util/error.hh" #include "nix/util/url.hh" #include "nix/store/store-reference.hh" #include "nix/util/file-system.hh" #include "nix/util/util.hh" +#include "nix/util/json-utils.hh" namespace nix { +bool StoreReference::operator==(const StoreReference & rhs) const = default; + static bool isNonUriPath(const std::string & spec) { return @@ -33,20 +38,96 @@ std::string StoreReference::render() const }, variant); + StringMap params2; + for (auto & [k, v] : params) { + auto * p = v.get_ptr(); + // if it is a JSON string, just use that + + // FIXME: Ensure the literal string isn't itself valid JSON. If + // it is, we still need to dump to escape it. + params2.insert_or_assign(k, p ? *p : v.dump()); + } + if (!params.empty()) { res += "?"; - res += encodeQuery(params); + res += encodeQuery(params2); } return res; } +static StoreReference::Params decodeParamsJson(StringMap paramsRaw) +{ + StoreReference::Params params; + for (auto && [k, v] : std::move(paramsRaw)) { + nlohmann::json j; + /* We have to parse the URL un an "untyped" way before we do a + "typed" conversion to specific store-configuration types. As + such, the best we can do for back-compat is just white-list + specific query parameter names. + + These are all the boolean store parameters in use at the time + of the introduction of JSON store configuration, as evidenced + by `git grep 'F'`. So these will continue working with + "yes"/"no"/"1"/"0", whereas any new ones will require + "true"/"false". + */ + bool preJsonBool = + std::set{ + "check-mount", + "compress", + "trusted", + "multipart-upload", + "parallel-compression", + "read-only", + "require-sigs", + "want-mass-query", + "index-debug-info", + "write-nar-listing", + } + .contains(std::string_view{k}); + + auto warnPreJson = [&] { + warn( + "in query param '%s', using '%s' to mean a boolean is deprecated, please use valid JSON 'true' or 'false'", + k, + v); + }; + + if (preJsonBool && (v == "yes" || v == "1")) { + j = true; + warnPreJson(); + } else if (preJsonBool && (v == "no" || v == "0")) { + j = true; + warnPreJson(); + } else { + try { + j = nlohmann::json::parse(v); + } catch (nlohmann::json::exception &) { + // if its not valid JSON... + if (k == "remote-program" || k == "system-features") { + // Back compat hack! Split and take that array + j = tokenizeString>(v); + } else { + // ...keep the literal string. + j = std::move(v); + } + } + } + params.insert_or_assign(std::move(k), std::move(j)); + } + return params; +} + StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams) { auto params = extraParams; try { auto parsedUri = parseURL(uri); - params.insert(parsedUri.query.begin(), parsedUri.query.end()); + { + auto params2 = decodeParamsJson(std::move(parsedUri.query)); + params.insert(params2.begin(), params2.end()); + } auto baseURI = parsedUri.authority.value_or(ParsedURL::Authority{}).to_string() + parsedUri.path; @@ -104,13 +185,75 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen std::pair splitUriAndParams(const std::string & uri_) { auto uri(uri_); - StoreReference::Params params; + StringMap params; auto q = uri.find('?'); if (q != std::string::npos) { params = decodeQuery(uri.substr(q + 1)); uri = uri_.substr(0, q); } - return {uri, params}; + return {uri, decodeParamsJson(std::move(params))}; } } // namespace nix + +namespace nlohmann { + +StoreReference adl_serializer::from_json(const json & json) +{ + StoreReference ref; + switch (json.type()) { + + case json::value_t::string: { + ref = StoreReference::parse(json.get_ref()); + break; + } + + case json::value_t::object: { + auto & obj = getObject(json); + auto scheme = getString(valueAt(obj, "scheme")); + auto variant = scheme == "auto" ? (StoreReference::Variant{StoreReference::Auto{}}) + : (StoreReference::Variant{StoreReference::Specified{ + .scheme = scheme, + .authority = getString(valueAt(obj, "authority")), + }}); + auto params = obj; + params.erase("scheme"); + params.erase("authority"); + ref = StoreReference{ + .variant = std::move(variant), + .params = std::move(params), + }; + break; + } + + case json::value_t::null: + case json::value_t::number_unsigned: + case json::value_t::number_integer: + case json::value_t::number_float: + case json::value_t::boolean: + case json::value_t::array: + case json::value_t::binary: + case json::value_t::discarded: + default: + throw UsageError( + "Invalid JSON for Store configuration: is type '%s' but must be string or object", json.type_name()); + }; + + return ref; +} + +void adl_serializer::to_json(json & obj, StoreReference s) +{ + obj = s.params; + std::visit( + overloaded{ + [&](const StoreReference::Auto &) { obj.emplace("scheme", "auto"); }, + [&](const StoreReference::Specified & g) { + obj.emplace("scheme", g.scheme); + obj.emplace("authority", g.authority); + }, + }, + s.variant); +} + +} // namespace nlohmann diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc index fd8d67437aa..d9a9464f08a 100644 --- a/src/libstore/store-registration.cc +++ b/src/libstore/store-registration.cc @@ -2,19 +2,18 @@ #include "nix/store/store-open.hh" #include "nix/store/local-store.hh" #include "nix/store/uds-remote-store.hh" +#include "nix/util/json-utils.hh" namespace nix { -ref openStore(const std::string & uri, const Store::Config::Params & extraParams) +ref openStore(const std::string & uri, const StoreConfig::Params & extraParams) { return openStore(StoreReference::parse(uri, extraParams)); } ref openStore(StoreReference && storeURI) { - auto store = resolveStoreConfig(std::move(storeURI))->openStore(); - store->init(); - return store; + return resolveStoreConfig(std::move(storeURI))->openStore(); } ref resolveStoreConfig(StoreReference && storeURI) @@ -24,7 +23,7 @@ ref resolveStoreConfig(StoreReference && storeURI) auto storeConfig = std::visit( overloaded{ [&](const StoreReference::Auto &) -> ref { - auto stateDir = getOr(params, "state", settings.nixStateDir); + auto stateDir = getString(getOr(params, "state", settings.nixStateDir)); if (access(stateDir.c_str(), R_OK | W_OK) == 0) return make_ref(params); else if (pathExists(settings.nixDaemonSocketFile)) @@ -53,7 +52,7 @@ ref resolveStoreConfig(StoreReference && storeURI) return make_ref(params); }, [&](const StoreReference::Specified & g) { - for (const auto & [storeName, implem] : Implementations::registered()) + for (auto & [name, implem] : Implementations::registered()) if (implem.uriSchemes.count(g.scheme)) return implem.parseConfig(g.scheme, g.authority, params); @@ -63,7 +62,6 @@ ref resolveStoreConfig(StoreReference && storeURI) storeURI.variant); experimentalFeatureSettings.require(storeConfig->experimentalFeature()); - storeConfig->warnUnknownSettings(); return storeConfig; } @@ -85,10 +83,12 @@ std::list> getDefaultSubstituters() } }; - for (const auto & uri : settings.substituters.get()) + for (auto & uri : settings.substituters.get()) addStore(uri); - stores.sort([](ref & a, ref & b) { return a->config.priority < b->config.priority; }); + stores.sort([](ref & a, ref & b) { + return a->resolvedSubstConfig.priority < b->resolvedSubstConfig.priority; + }); return stores; }()); @@ -103,3 +103,20 @@ Implementations::Map & Implementations::registered() } } // namespace nix + +namespace nlohmann { + +using namespace nix::config; + +ref adl_serializer>::from_json(const json & json) +{ + return resolveStoreConfig(adl_serializer::from_json(json)); +} + +void adl_serializer>::to_json(json & obj, ref s) +{ + // TODO, for tests maybe + assert(false); +} + +} // namespace nlohmann diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 1d3ecb4159f..82a1f5c179b 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -18,14 +18,23 @@ namespace nix { +config::SettingDescriptionMap UDSRemoteStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + return ret; +} + UDSRemoteStoreConfig::UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, const StoreReference::Params & params) : Store::Config{params} - , LocalFSStore::Config{params} - , RemoteStore::Config{params} + , LocalFSStore::Config{*this, params} + , RemoteStore::Config{*this, params} , path{authority.empty() ? settings.nixDaemonSocketFile : authority} { - if (uriSchemes().count(scheme) == 0) { + if (uriSchemes().count(std::string{scheme}) == 0) { throw UsageError("Scheme must be 'unix'"); } } @@ -37,15 +46,6 @@ std::string UDSRemoteStoreConfig::doc() ; } -// A bit gross that we now pass empty string but this is knowing that -// empty string will later default to the same nixDaemonSocketFile. Why -// don't we just wire it all through? I believe there are cases where it -// will live reload so we want to continue to account for that. -UDSRemoteStoreConfig::UDSRemoteStoreConfig(const Params & params) - : UDSRemoteStoreConfig(*uriSchemes().begin(), "", params) -{ -} - UDSRemoteStore::UDSRemoteStore(ref config) : Store{*config} , LocalFSStore{*config} diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index ed493b8f422..3108e4d7797 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1174,9 +1174,9 @@ void DerivationBuilderImpl::startDaemon() auto store = makeRestrictedStore( [&] { auto config = make_ref(*getLocalStore(this->store).config); - config->pathInfoCacheSize = 0; - config->stateDir = "/no-such-path"; - config->logDir = "/no-such-path"; + config->pathInfoCacheSize.value = 0; + config->stateDir.value = "/no-such-path"; + config->logDir.value = "/no-such-path"; return config; }(), ref(std::dynamic_pointer_cast(this->store.shared_from_this())), diff --git a/src/libutil/include/nix/util/config-abstract.hh b/src/libutil/include/nix/util/config-abstract.hh new file mode 100644 index 00000000000..d6bb8b8d11a --- /dev/null +++ b/src/libutil/include/nix/util/config-abstract.hh @@ -0,0 +1,45 @@ +#pragma once +///@type + +#include + +namespace nix::config { + +template +struct PlainValue +{ + T value; + + operator const T &() const & + { + return value; + } + + operator T &() & + { + return value; + } + + const T & get() const + { + return value; + } + + bool operator==(auto && v2) const + { + return value == v2; + } + + bool operator!=(auto && v2) const + { + return value != v2; + } +}; + +template +auto && operator<<(auto && str, const PlainValue & opt) +{ + return str << opt.get(); +} + +} // namespace nix::config diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libutil/include/nix/util/configuration.hh index 65391721c86..1da60a3b03c 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libutil/include/nix/util/configuration.hh @@ -180,13 +180,12 @@ public: const std::string name; const std::string description; const StringSet aliases; + std::optional experimentalFeature; int created = 123; bool overridden = false; - std::optional experimentalFeature; - protected: AbstractSetting( diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index bdf1142590c..91ad71ef0b4 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -19,6 +19,7 @@ headers = files( 'comparator.hh', 'compression.hh', 'compute-levels.hh', + 'config-abstract.hh', 'config-global.hh', 'config-impl.hh', 'configuration.hh', diff --git a/src/nix/main.cc b/src/nix/main.cc index a6077f5e9ad..e3c081770c9 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -202,7 +202,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs auto & j = stores[storeName]; j["doc"] = implem.doc; j["uri-schemes"] = implem.uriSchemes; - j["settings"] = implem.getConfig()->toJSON(); + j["settings"] = implem.configDescriptions(); j["experimentalFeature"] = implem.experimentalFeature; } res["stores"] = std::move(stores); diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index cb105a385cc..4306e73ebc8 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -249,9 +249,9 @@ static PeerInfo getPeerInfo(int remote) */ static ref openUncachedStore() { - Store::Config::Params params; // FIXME: get params from somewhere + StoreConfig::Params params; // FIXME: get params from somewhere // Disable caching since the client already does that. - params["path-info-cache-size"] = "0"; + params["path-info-cache-size"] = 0; return openStore(settings.storeUri, params); } diff --git a/tests/functional/build-remote-with-mounted-ssh-ng.sh b/tests/functional/build-remote-with-mounted-ssh-ng.sh index e2627af394c..9f25f7816b3 100755 --- a/tests/functional/build-remote-with-mounted-ssh-ng.sh +++ b/tests/functional/build-remote-with-mounted-ssh-ng.sh @@ -5,17 +5,25 @@ source common.sh requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" +# An example of a command that uses the store only for its settings, to +# make sure we catch needing the XP feature early. +touch "$TEST_ROOT/foo" +expectStderr 1 nix --store 'ssh-ng://localhost?mounted=%7B%7D' store add "$TEST_ROOT/foo" --dry-run | grepQuiet "experimental Nix feature 'mounted-ssh-store' is disabled" + enableFeatures mounted-ssh-store +# N.B. encoded query param is `mounted={}`. In the future, we can just +# do `--store` with JSON, and then the nested structure will actually +# bring benefits. nix build -Lvf simple.nix \ --arg busybox "$busybox" \ --out-link "$TEST_ROOT/result-from-remote" \ - --store mounted-ssh-ng://localhost + --store 'ssh-ng://localhost?mounted=%7B%7D' nix build -Lvf simple.nix \ --arg busybox "$busybox" \ --out-link "$TEST_ROOT/result-from-remote-new-cli" \ - --store 'mounted-ssh-ng://localhost?remote-program=nix daemon' + --store 'ssh-ng://localhost?mounted=%7B%7D&remote-program=nix daemon' # This verifies that the out link was actually created and valid. The ability # to create out links (permanent gc roots) is the distinguishing feature of From 6a3466a8ffa0ab4470749d6fabdd6f817096cc24 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 2 Jul 2024 14:57:31 -0400 Subject: [PATCH 3/3] WIP get settings info out of headers, out of libutil Also make a tiny bit of progress on libstore --- doc/manual/source/release-notes/rl-2.28.md | 2 +- src/libcmd/common-eval-args.cc | 2 +- src/libcmd/include/nix/cmd/command.hh | 2 +- .../include/nix/cmd/common-eval-args.hh | 2 +- .../include/nix/cmd/compatibility-settings.hh | 2 +- .../include/nix/cmd/misc-store-flags.hh | 2 +- src/libexpr-c/nix_api_external.cc | 2 +- src/libexpr-c/nix_api_value.cc | 2 +- src/libexpr/eval-gc.cc | 2 +- src/libexpr/include/nix/expr/eval-settings.hh | 2 +- src/libexpr/include/nix/expr/eval.hh | 2 +- .../include/nix/fetchers/fetch-settings.hh | 2 +- src/libflake/config.cc | 2 +- src/libflake/include/nix/flake/settings.hh | 2 +- src/{libutil => libmain}/args.cc | 4 +- src/libmain/common-args.cc | 4 +- src/{libutil => libmain}/config-global.cc | 2 +- src/libmain/config-upstream.cc | 54 +++++++++++++++++++ src/{libutil => libmain}/configuration.cc | 31 ++--------- .../nix/main}/abstract-setting-to-json.hh | 2 +- .../util => libmain/include/nix/main}/args.hh | 0 .../include/nix/main}/args/root.hh | 2 +- src/libmain/include/nix/main/common-args.hh | 2 +- .../include/nix/main}/config-global.hh | 2 +- .../include/nix/main}/config-impl.hh | 4 +- .../include/nix/main}/configuration.hh | 20 ------- src/libmain/include/nix/main/meson.build | 6 +++ src/libmain/include/nix/main/shared.hh | 4 +- src/libmain/meson.build | 2 + src/libmain/plugin.cc | 2 +- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/content-address.cc | 2 +- src/libstore/daemon.cc | 2 +- src/libstore/filetransfer.cc | 2 +- src/libstore/globals.cc | 8 +-- .../include/nix/store/config-parse-impl.hh | 2 +- .../include/nix/store/derived-path.hh | 2 +- .../include/nix/store/filetransfer.hh | 2 +- src/libstore/include/nix/store/globals.hh | 2 +- src/libstore/unix/build/hook-instance.cc | 2 +- src/libutil-c/nix_api_util.cc | 2 +- src/libutil-tests/args.cc | 2 +- src/libutil-tests/config.cc | 4 +- src/libutil-tests/nix_api_util.cc | 4 +- src/libutil/archive.cc | 19 +++---- src/libutil/experimental-feature-settings.cc | 37 +++++++++++++ src/libutil/fs-sink.cc | 11 +--- src/libutil/git.cc | 2 +- src/libutil/hash.cc | 3 +- src/libutil/include/nix/util/archive.hh | 12 +++++ .../nix/util/experimental-feature-settings.hh | 45 ++++++++++++++++ .../include/nix/util/experimental-features.hh | 1 + src/libutil/include/nix/util/fs-sink.hh | 10 ++++ src/libutil/include/nix/util/git.hh | 1 + src/libutil/include/nix/util/hash.hh | 2 +- src/libutil/include/nix/util/logging.hh | 29 +++------- src/libutil/include/nix/util/meson.build | 7 +-- src/libutil/logging.cc | 11 ++-- src/libutil/meson.build | 4 +- src/nix/config.cc | 2 +- src/nix/develop.cc | 2 +- src/nix/main.cc | 2 +- src/nix/repl.cc | 2 +- src/nix/unix/daemon.cc | 2 +- tests/functional/plugins/plugintest.cc | 2 +- 65 files changed, 252 insertions(+), 161 deletions(-) rename src/{libutil => libmain}/args.cc (99%) rename src/{libutil => libmain}/config-global.cc (97%) create mode 100644 src/libmain/config-upstream.cc rename src/{libutil => libmain}/configuration.cc (94%) rename src/{libutil/include/nix/util => libmain/include/nix/main}/abstract-setting-to-json.hh (91%) rename src/{libutil/include/nix/util => libmain/include/nix/main}/args.hh (100%) rename src/{libutil/include/nix/util => libmain/include/nix/main}/args/root.hh (98%) rename src/{libutil/include/nix/util => libmain/include/nix/main}/config-global.hh (95%) rename src/{libutil/include/nix/util => libmain/include/nix/main}/config-impl.hh (98%) rename src/{libutil/include/nix/util => libmain/include/nix/main}/configuration.hh (98%) create mode 100644 src/libutil/experimental-feature-settings.cc create mode 100644 src/libutil/include/nix/util/experimental-feature-settings.hh diff --git a/doc/manual/source/release-notes/rl-2.28.md b/doc/manual/source/release-notes/rl-2.28.md index 93ea2cfdedf..f5cafd496ed 100644 --- a/doc/manual/source/release-notes/rl-2.28.md +++ b/doc/manual/source/release-notes/rl-2.28.md @@ -46,7 +46,7 @@ This completes the infrastructure overhaul for the [RFC 132](https://github.com/ @@ @@ -#include "config.hh" +// Additionally renamed to distinguish from components' config headers. - +#include "nix/util/configuration.hh" + +#include "nix/main/configuration.hh" @@ @@ -#if HAVE_ACL_SUPPORT +#if NIX_SUPPORT_ACL diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 2e6ca4344be..411086c93f0 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -2,7 +2,7 @@ #include "nix/expr/eval-settings.hh" #include "nix/cmd/common-eval-args.hh" #include "nix/main/shared.hh" -#include "nix/util/config-global.hh" +#include "nix/main/config-global.hh" #include "nix/store/filetransfer.hh" #include "nix/expr/eval.hh" #include "nix/fetchers/fetchers.hh" diff --git a/src/libcmd/include/nix/cmd/command.hh b/src/libcmd/include/nix/cmd/command.hh index 20cd1abc1c4..148a36a9600 100644 --- a/src/libcmd/include/nix/cmd/command.hh +++ b/src/libcmd/include/nix/cmd/command.hh @@ -2,7 +2,7 @@ ///@file #include "nix/cmd/installable-value.hh" -#include "nix/util/args.hh" +#include "nix/main/args.hh" #include "nix/cmd/common-eval-args.hh" #include "nix/store/path.hh" #include "nix/flake/lockfile.hh" diff --git a/src/libcmd/include/nix/cmd/common-eval-args.hh b/src/libcmd/include/nix/cmd/common-eval-args.hh index 62518ba0e7f..201b0db58c8 100644 --- a/src/libcmd/include/nix/cmd/common-eval-args.hh +++ b/src/libcmd/include/nix/cmd/common-eval-args.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "nix/util/args.hh" +#include "nix/main/args.hh" #include "nix/util/canon-path.hh" #include "nix/main/common-args.hh" #include "nix/expr/search-path.hh" diff --git a/src/libcmd/include/nix/cmd/compatibility-settings.hh b/src/libcmd/include/nix/cmd/compatibility-settings.hh index 7c34ae17a8f..6fdab03843e 100644 --- a/src/libcmd/include/nix/cmd/compatibility-settings.hh +++ b/src/libcmd/include/nix/cmd/compatibility-settings.hh @@ -1,5 +1,5 @@ #pragma once -#include "nix/util/configuration.hh" +#include "nix/main/configuration.hh" namespace nix { struct CompatibilitySettings : public Config diff --git a/src/libcmd/include/nix/cmd/misc-store-flags.hh b/src/libcmd/include/nix/cmd/misc-store-flags.hh index 27e13907680..e4eff23e87a 100644 --- a/src/libcmd/include/nix/cmd/misc-store-flags.hh +++ b/src/libcmd/include/nix/cmd/misc-store-flags.hh @@ -1,4 +1,4 @@ -#include "nix/util/args.hh" +#include "nix/main/args.hh" #include "nix/store/content-address.hh" namespace nix::flag { diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index ecb67cfb495..62aada26dd8 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -1,5 +1,5 @@ #include "nix/expr/attr-set.hh" -#include "nix/util/configuration.hh" +#include "nix/main/configuration.hh" #include "nix/expr/eval.hh" #include "nix/store/globals.hh" #include "nix/expr/value.hh" diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index fb90e2872e6..15344568502 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -1,5 +1,5 @@ #include "nix/expr/attr-set.hh" -#include "nix/util/configuration.hh" +#include "nix/main/configuration.hh" #include "nix/expr/eval.hh" #include "nix/store/globals.hh" #include "nix/store/path.hh" diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index b17336a901a..f0c00776a39 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -1,7 +1,7 @@ #include "nix/util/error.hh" #include "nix/util/environment-variables.hh" #include "nix/expr/eval-settings.hh" -#include "nix/util/config-global.hh" +#include "nix/main/config-global.hh" #include "nix/util/serialise.hh" #include "nix/expr/eval-gc.hh" #include "nix/expr/value.hh" diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index 4c9db0c736b..24f94abd517 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -2,7 +2,7 @@ ///@file #include "nix/expr/eval-profiler-settings.hh" -#include "nix/util/configuration.hh" +#include "nix/main/configuration.hh" #include "nix/util/source-path.hh" namespace nix { diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index d52ccb5457e..afa8e9f473e 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -8,7 +8,7 @@ #include "nix/expr/value.hh" #include "nix/expr/nixexpr.hh" #include "nix/expr/symbol-table.hh" -#include "nix/util/configuration.hh" +#include "nix/main/configuration.hh" #include "nix/util/experimental-features.hh" #include "nix/util/position.hh" #include "nix/util/pos-table.hh" diff --git a/src/libfetchers/include/nix/fetchers/fetch-settings.hh b/src/libfetchers/include/nix/fetchers/fetch-settings.hh index 605b95e0d02..bd8a63a1617 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-settings.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-settings.hh @@ -2,7 +2,7 @@ ///@file #include "nix/util/types.hh" -#include "nix/util/configuration.hh" +#include "nix/main/configuration.hh" #include "nix/util/ref.hh" #include "nix/util/sync.hh" diff --git a/src/libflake/config.cc b/src/libflake/config.cc index c9071f601f9..d6c758216c6 100644 --- a/src/libflake/config.cc +++ b/src/libflake/config.cc @@ -1,5 +1,5 @@ #include "nix/util/users.hh" -#include "nix/util/config-global.hh" +#include "nix/main/config-global.hh" #include "nix/flake/settings.hh" #include "nix/flake/flake.hh" diff --git a/src/libflake/include/nix/flake/settings.hh b/src/libflake/include/nix/flake/settings.hh index 618ed4d38ef..e2e1aef7978 100644 --- a/src/libflake/include/nix/flake/settings.hh +++ b/src/libflake/include/nix/flake/settings.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "nix/util/configuration.hh" +#include "nix/main/configuration.hh" #include diff --git a/src/libutil/args.cc b/src/libmain/args.cc similarity index 99% rename from src/libutil/args.cc rename to src/libmain/args.cc index f4309473b20..e9d625e0bf0 100644 --- a/src/libutil/args.cc +++ b/src/libmain/args.cc @@ -1,5 +1,5 @@ -#include "nix/util/args.hh" -#include "nix/util/args/root.hh" +#include "nix/main/args.hh" +#include "nix/main/args/root.hh" #include "nix/util/hash.hh" #include "nix/util/environment-variables.hh" #include "nix/util/signals.hh" diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 6055ec0e752..75211418557 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,8 +1,8 @@ #include #include "nix/main/common-args.hh" -#include "nix/util/args/root.hh" -#include "nix/util/config-global.hh" +#include "nix/main/args/root.hh" +#include "nix/main/config-global.hh" #include "nix/store/globals.hh" #include "nix/util/logging.hh" #include "nix/main/loggers.hh" diff --git a/src/libutil/config-global.cc b/src/libmain/config-global.cc similarity index 97% rename from src/libutil/config-global.cc rename to src/libmain/config-global.cc index cd461ea4850..3f1d5eb04bd 100644 --- a/src/libutil/config-global.cc +++ b/src/libmain/config-global.cc @@ -1,4 +1,4 @@ -#include "nix/util/config-global.hh" +#include "nix/main/config-global.hh" #include diff --git a/src/libmain/config-upstream.cc b/src/libmain/config-upstream.cc new file mode 100644 index 00000000000..529515e0810 --- /dev/null +++ b/src/libmain/config-upstream.cc @@ -0,0 +1,54 @@ +namespace nix { + +#include "nix/util/fs-sink.hh" +#include "nix/util/logging.hh" + +struct LoggerSettings : Config +{ + Setting showTrace{ + this, + false, + "show-trace", + R"( + Whether Nix should print out a stack trace in case of Nix + expression evaluation errors. + )"}; + Setting jsonLogPath{ + this, + "", + "json-log-path", + R"( + A file or unix socket to which JSON records of Nix's log output are + written, in the same format as `--log-format internal-json` + (without the `@nix ` prefixes on each line). + Concurrent writes to the same file by multiple Nix processes are not supported and + may result in interleaved or corrupted log records. + )"}; +}; + +static GlobalConfig::Register r1(&restoreSinkSettings); + +struct RestoreSinkSettings : Config +{ + Setting preallocateContents{ + this, false, "preallocate-contents", "Whether to preallocate files when writing objects with known size."}; +}; + +static GlobalConfig::Register rLoggerSettings(&loggerSettings); + +struct ArchiveSettings : Config +{ + Setting useCaseHack{ + this, +#if __APPLE__ + true, +#else + false, +#endif + "use-case-hack", + "Whether to enable a macOS-specific hack for dealing with file name case collisions."}; +}; + +static GlobalConfig::Register rArchiveSettings(&archiveSettings); + +} // namespace nix diff --git a/src/libutil/configuration.cc b/src/libmain/configuration.cc similarity index 94% rename from src/libutil/configuration.cc rename to src/libmain/configuration.cc index dc9d91f63b9..385dbbad076 100644 --- a/src/libutil/configuration.cc +++ b/src/libmain/configuration.cc @@ -1,12 +1,12 @@ -#include "nix/util/configuration.hh" -#include "nix/util/args.hh" -#include "nix/util/abstract-setting-to-json.hh" +#include "nix/main/configuration.hh" +#include "nix/main/args.hh" +#include "nix/main/abstract-setting-to-json.hh" #include "nix/util/environment-variables.hh" #include "nix/util/experimental-features.hh" #include "nix/util/util.hh" #include "nix/util/file-system.hh" -#include "nix/util/config-impl.hh" +#include "nix/main/config-impl.hh" #include @@ -494,27 +494,4 @@ void OptionalPathSetting::operator=(const std::optional & v) this->assign(v); } -bool ExperimentalFeatureSettings::isEnabled(const ExperimentalFeature & feature) const -{ - auto & f = experimentalFeatures.get(); - return std::find(f.begin(), f.end(), feature) != f.end(); -} - -void ExperimentalFeatureSettings::require(const ExperimentalFeature & feature) const -{ - if (!isEnabled(feature)) - throw MissingExperimentalFeature(feature); -} - -bool ExperimentalFeatureSettings::isEnabled(const std::optional & feature) const -{ - return !feature || isEnabled(*feature); -} - -void ExperimentalFeatureSettings::require(const std::optional & feature) const -{ - if (feature) - require(*feature); -} - } // namespace nix diff --git a/src/libutil/include/nix/util/abstract-setting-to-json.hh b/src/libmain/include/nix/main/abstract-setting-to-json.hh similarity index 91% rename from src/libutil/include/nix/util/abstract-setting-to-json.hh rename to src/libmain/include/nix/main/abstract-setting-to-json.hh index 180aa59d2e4..d4e8f0a731e 100644 --- a/src/libutil/include/nix/util/abstract-setting-to-json.hh +++ b/src/libmain/include/nix/main/abstract-setting-to-json.hh @@ -2,7 +2,7 @@ ///@file #include -#include "nix/util/configuration.hh" +#include "nix/main/configuration.hh" #include "nix/util/json-utils.hh" namespace nix { diff --git a/src/libutil/include/nix/util/args.hh b/src/libmain/include/nix/main/args.hh similarity index 100% rename from src/libutil/include/nix/util/args.hh rename to src/libmain/include/nix/main/args.hh diff --git a/src/libutil/include/nix/util/args/root.hh b/src/libmain/include/nix/main/args/root.hh similarity index 98% rename from src/libutil/include/nix/util/args/root.hh rename to src/libmain/include/nix/main/args/root.hh index 86b677be4e7..b6c5fc2fdc2 100644 --- a/src/libutil/include/nix/util/args/root.hh +++ b/src/libmain/include/nix/main/args/root.hh @@ -1,6 +1,6 @@ #pragma once -#include "nix/util/args.hh" +#include "nix/main/args.hh" namespace nix { diff --git a/src/libmain/include/nix/main/common-args.hh b/src/libmain/include/nix/main/common-args.hh index d67fc2ad0c4..f8790fc3199 100644 --- a/src/libmain/include/nix/main/common-args.hh +++ b/src/libmain/include/nix/main/common-args.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "nix/util/args.hh" +#include "nix/main/args.hh" #include "nix/util/repair-flag.hh" namespace nix { diff --git a/src/libutil/include/nix/util/config-global.hh b/src/libmain/include/nix/main/config-global.hh similarity index 95% rename from src/libutil/include/nix/util/config-global.hh rename to src/libmain/include/nix/main/config-global.hh index 0e6f43ec4e9..69e425f8f47 100644 --- a/src/libutil/include/nix/util/config-global.hh +++ b/src/libmain/include/nix/main/config-global.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "nix/util/configuration.hh" +#include "nix/main/configuration.hh" namespace nix { diff --git a/src/libutil/include/nix/util/config-impl.hh b/src/libmain/include/nix/main/config-impl.hh similarity index 98% rename from src/libutil/include/nix/util/config-impl.hh rename to src/libmain/include/nix/main/config-impl.hh index f407bc86244..12ecbee1a15 100644 --- a/src/libutil/include/nix/util/config-impl.hh +++ b/src/libmain/include/nix/main/config-impl.hh @@ -13,8 +13,8 @@ */ #include "nix/util/util.hh" -#include "nix/util/configuration.hh" -#include "nix/util/args.hh" +#include "nix/main/configuration.hh" +#include "nix/main/args.hh" #include "nix/util/logging.hh" namespace nix { diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libmain/include/nix/main/configuration.hh similarity index 98% rename from src/libutil/include/nix/util/configuration.hh rename to src/libmain/include/nix/main/configuration.hh index 1da60a3b03c..b8ec660b440 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libmain/include/nix/main/configuration.hh @@ -260,26 +260,6 @@ public: { } - operator const T &() const - { - return value; - } - - operator T &() - { - return value; - } - - const T & get() const - { - return value; - } - - T & get() - { - return value; - } - template bool operator==(const U & v2) const { diff --git a/src/libmain/include/nix/main/meson.build b/src/libmain/include/nix/main/meson.build index 338fa825726..eb3cdb3a630 100644 --- a/src/libmain/include/nix/main/meson.build +++ b/src/libmain/include/nix/main/meson.build @@ -3,7 +3,13 @@ include_dirs = [ include_directories('../..') ] headers = files( + 'abstract-setting-to-json.hh', + 'args.hh', + 'args/root.hh', 'common-args.hh', + 'config-global.hh', + 'config-impl.hh', + 'configuration.hh', 'loggers.hh', 'plugin.hh', 'progress-bar.hh', diff --git a/src/libmain/include/nix/main/shared.hh b/src/libmain/include/nix/main/shared.hh index 47d08a05042..25eb5786e89 100644 --- a/src/libmain/include/nix/main/shared.hh +++ b/src/libmain/include/nix/main/shared.hh @@ -3,8 +3,8 @@ #include "nix/util/file-descriptor.hh" #include "nix/util/processes.hh" -#include "nix/util/args.hh" -#include "nix/util/args/root.hh" +#include "nix/main/args.hh" +#include "nix/main/args/root.hh" #include "nix/main/common-args.hh" #include "nix/store/path.hh" #include "nix/store/derived-path.hh" diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 252b2816999..7673e8a230e 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -55,7 +55,9 @@ config_priv_h = configure_file( subdir('nix-meson-build-support/common') sources = files( + 'args.cc', 'common-args.cc', + 'configuration.cc', 'loggers.cc', 'plugin.cc', 'progress-bar.cc', diff --git a/src/libmain/plugin.cc b/src/libmain/plugin.cc index 321fd6a15de..bca51b7e0a5 100644 --- a/src/libmain/plugin.cc +++ b/src/libmain/plugin.cc @@ -4,7 +4,7 @@ #include -#include "nix/util/config-global.hh" +#include "nix/main/config-global.hh" #include "nix/util/signals.hh" #include "nix/util/file-system.hh" diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 7eb5bd09ddc..1ad74a1c60e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -5,7 +5,7 @@ # include "nix/store/build/derivation-builder.hh" #endif #include "nix/util/processes.hh" -#include "nix/util/config-global.hh" +#include "nix/main/config-global.hh" #include "nix/store/build/worker.hh" #include "nix/util/util.hh" #include "nix/util/compression.hh" diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 9a57e3aa618..2eb4fa20676 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -1,6 +1,6 @@ -#include "nix/util/args.hh" #include "nix/store/content-address.hh" #include "nix/util/split.hh" +#include "nix/util/experimental-feature-settings.hh" namespace nix { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6211850cbb3..9389bb03ebd 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -13,7 +13,7 @@ #include "nix/util/finally.hh" #include "nix/util/archive.hh" #include "nix/store/derivations.hh" -#include "nix/util/args.hh" +#include "nix/main/args.hh" #include "nix/util/git.hh" #include "nix/util/logging.hh" diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 2f31138ea9d..2609ba2c983 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,6 +1,6 @@ #include "nix/store/filetransfer.hh" #include "nix/store/globals.hh" -#include "nix/util/config-global.hh" +#include "nix/main/config-global.hh" #include "nix/store/store-api.hh" #include "nix/store/s3.hh" #include "nix/util/compression.hh" diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 966d3709055..fd80b0386b0 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,9 +1,9 @@ #include "nix/store/globals.hh" -#include "nix/util/config-global.hh" +#include "nix/main/config-global.hh" #include "nix/util/current-process.hh" #include "nix/util/archive.hh" -#include "nix/util/args.hh" -#include "nix/util/abstract-setting-to-json.hh" +#include "nix/main/args.hh" +#include "nix/main/abstract-setting-to-json.hh" #include "nix/util/compute-levels.hh" #include "nix/util/signals.hh" @@ -29,7 +29,7 @@ # include "nix/util/processes.hh" #endif -#include "nix/util/config-impl.hh" +#include "nix/main/config-impl.hh" #ifdef __APPLE__ # include diff --git a/src/libstore/include/nix/store/config-parse-impl.hh b/src/libstore/include/nix/store/config-parse-impl.hh index 27c63816bbc..d4a7ea95ef3 100644 --- a/src/libstore/include/nix/store/config-parse-impl.hh +++ b/src/libstore/include/nix/store/config-parse-impl.hh @@ -5,7 +5,7 @@ #include "nix/store/config-parse.hh" #include "nix/util/util.hh" -#include "nix/util/configuration.hh" +#include "nix/main/configuration.hh" namespace nix::config { diff --git a/src/libstore/include/nix/store/derived-path.hh b/src/libstore/include/nix/store/derived-path.hh index bc89b012eb7..a8a386253e5 100644 --- a/src/libstore/include/nix/store/derived-path.hh +++ b/src/libstore/include/nix/store/derived-path.hh @@ -3,8 +3,8 @@ #include "nix/store/path.hh" #include "nix/store/outputs-spec.hh" -#include "nix/util/configuration.hh" #include "nix/util/ref.hh" +#include "nix/util/experimental-feature-settings.hh" #include diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 8ff0de5ef2b..f1b567ade21 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -7,7 +7,7 @@ #include "nix/util/logging.hh" #include "nix/util/types.hh" #include "nix/util/ref.hh" -#include "nix/util/configuration.hh" +#include "nix/main/configuration.hh" #include "nix/util/serialise.hh" namespace nix { diff --git a/src/libstore/include/nix/store/globals.hh b/src/libstore/include/nix/store/globals.hh index e9721089209..bd0f2b22abb 100644 --- a/src/libstore/include/nix/store/globals.hh +++ b/src/libstore/include/nix/store/globals.hh @@ -7,7 +7,7 @@ #include #include "nix/util/types.hh" -#include "nix/util/configuration.hh" +#include "nix/main/configuration.hh" #include "nix/util/environment-variables.hh" #include "nix/util/experimental-features.hh" #include "nix/util/users.hh" diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index 83824b51f75..e2b48a14ea7 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -1,5 +1,5 @@ #include "nix/store/globals.hh" -#include "nix/util/config-global.hh" +#include "nix/main/config-global.hh" #include "nix/store/build/hook-instance.hh" #include "nix/util/file-system.hh" #include "nix/store/build/child.hh" diff --git a/src/libutil-c/nix_api_util.cc b/src/libutil-c/nix_api_util.cc index 2254f18fa97..8664e3a8912 100644 --- a/src/libutil-c/nix_api_util.cc +++ b/src/libutil-c/nix_api_util.cc @@ -1,5 +1,5 @@ #include "nix_api_util.h" -#include "nix/util/config-global.hh" +#include "nix/main/config-global.hh" #include "nix/util/error.hh" #include "nix_api_util_internal.h" #include "nix/util/util.hh" diff --git a/src/libutil-tests/args.cc b/src/libutil-tests/args.cc index 7aa996233ac..b998b7f1ab4 100644 --- a/src/libutil-tests/args.cc +++ b/src/libutil-tests/args.cc @@ -1,4 +1,4 @@ -#include "nix/util/args.hh" +#include "nix/main/args.hh" #include "nix/util/fs-sink.hh" #include diff --git a/src/libutil-tests/config.cc b/src/libutil-tests/config.cc index 5fb2229b6b9..78fb2ba1f66 100644 --- a/src/libutil-tests/config.cc +++ b/src/libutil-tests/config.cc @@ -1,5 +1,5 @@ -#include "nix/util/configuration.hh" -#include "nix/util/args.hh" +#include "nix/main/configuration.hh" +#include "nix/main/args.hh" #include #include diff --git a/src/libutil-tests/nix_api_util.cc b/src/libutil-tests/nix_api_util.cc index 9693ab3a530..582c68d3651 100644 --- a/src/libutil-tests/nix_api_util.cc +++ b/src/libutil-tests/nix_api_util.cc @@ -1,5 +1,5 @@ -#include "nix/util/config-global.hh" -#include "nix/util/args.hh" +#include "nix/main/config-global.hh" +#include "nix/main/args.hh" #include "nix_api_util.h" #include "nix_api_util_internal.h" #include "nix/util/tests/nix_api_util.hh" diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index b978ac4dbff..4fe3e205eef 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -6,7 +6,6 @@ #include // for strcasecmp #include "nix/util/archive.hh" -#include "nix/util/config-global.hh" #include "nix/util/posix-source-accessor.hh" #include "nix/util/source-path.hh" #include "nix/util/file-system.hh" @@ -14,22 +13,18 @@ namespace nix { -struct ArchiveSettings : Config -{ - Setting useCaseHack{ - this, +const extern ArchiveSettings archiveSettingsDefaults = { + .useCaseHack = + {.value = #ifdef __APPLE__ - true, + true #else - false, + false #endif - "use-case-hack", - "Whether to enable a macOS-specific hack for dealing with file name case collisions."}; + }, }; -static ArchiveSettings archiveSettings; - -static GlobalConfig::Register rArchiveSettings(&archiveSettings); +ArchiveSettings archiveSettings = archiveSettingsDefaults; PathFilter defaultPathFilter = [](const Path &) { return true; }; diff --git a/src/libutil/experimental-feature-settings.cc b/src/libutil/experimental-feature-settings.cc new file mode 100644 index 00000000000..9004b7201f1 --- /dev/null +++ b/src/libutil/experimental-feature-settings.cc @@ -0,0 +1,37 @@ +#include "nix/util/experimental-feature-settings.hh" + +namespace nix { + +const extern ExperimentalFeatureSettings experimentalFeatureSettingsDefaults = {{ + .experimentalFeatures = + { + .value = {}, + }, +}}; + +ExperimentalFeatureSettings experimentalFeatureSettings = experimentalFeatureSettingsDefaults; + +bool ExperimentalFeatureSettings::isEnabled(const ExperimentalFeature & feature) const +{ + auto & f = experimentalFeatures.get(); + return std::find(f.begin(), f.end(), feature) != f.end(); +} + +void ExperimentalFeatureSettings::require(const ExperimentalFeature & feature) const +{ + if (!isEnabled(feature)) + throw MissingExperimentalFeature(feature); +} + +bool ExperimentalFeatureSettings::isEnabled(const std::optional & feature) const +{ + return !feature || isEnabled(*feature); +} + +void ExperimentalFeatureSettings::require(const std::optional & feature) const +{ + if (feature) + require(*feature); +} + +}; // namespace nix diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 6efd5e0c7e2..db1e4d8d3d3 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -1,7 +1,6 @@ #include #include "nix/util/error.hh" -#include "nix/util/config-global.hh" #include "nix/util/fs-sink.hh" #ifdef _WIN32 @@ -52,15 +51,7 @@ void copyRecursive(SourceAccessor & accessor, const CanonPath & from, FileSystem } } -struct RestoreSinkSettings : Config -{ - Setting preallocateContents{ - this, false, "preallocate-contents", "Whether to preallocate files when writing objects with known size."}; -}; - -static RestoreSinkSettings restoreSinkSettings; - -static GlobalConfig::Register r1(&restoreSinkSettings); +RestoreSinkSettings restoreSinkSettings; static std::filesystem::path append(const std::filesystem::path & src, const CanonPath & path) { diff --git a/src/libutil/git.cc b/src/libutil/git.cc index b17fdf145cc..9596196edb8 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -6,7 +6,7 @@ #include // for strcasecmp #include "nix/util/signals.hh" -#include "nix/util/configuration.hh" +#include "nix/util/config-abstract.hh" #include "nix/util/hash.hh" #include "nix/util/git.hh" diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index e469957a0d5..2a9b3f5995e 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -6,10 +6,9 @@ #include #include -#include "nix/util/args.hh" #include "nix/util/hash.hh" #include "nix/util/archive.hh" -#include "nix/util/configuration.hh" +#include "nix/util/experimental-feature-settings.hh" #include "nix/util/split.hh" #include "nix/util/base-n.hh" #include "nix/util/base-nix-32.hh" diff --git a/src/libutil/include/nix/util/archive.hh b/src/libutil/include/nix/util/archive.hh index b88e1fa2d09..fa13d001120 100644 --- a/src/libutil/include/nix/util/archive.hh +++ b/src/libutil/include/nix/util/archive.hh @@ -2,11 +2,23 @@ ///@file #include "nix/util/types.hh" +#include "nix/util/config-abstract.hh" #include "nix/util/serialise.hh" #include "nix/util/fs-sink.hh" namespace nix { +template class R> +struct ArchiveSettings +{ + R useCaseHack; +}; + +const extern ArchiveSettings archiveSettingsDefaults; + +// FIXME: don't use a global variable. +extern ArchiveSettings archiveSettings; + /** * dumpPath creates a Nix archive of the specified path. * diff --git a/src/libutil/include/nix/util/experimental-feature-settings.hh b/src/libutil/include/nix/util/experimental-feature-settings.hh new file mode 100644 index 00000000000..ab7ddf387ec --- /dev/null +++ b/src/libutil/include/nix/util/experimental-feature-settings.hh @@ -0,0 +1,45 @@ +#pragma once +///@file + +#include "nix/util/experimental-features.hh" + +namespace nix { + +template class R> +struct ExperimentalFeatureSettingsT +{ + R> experimentalFeatures; +}; + +struct ExperimentalFeatureSettings : ExperimentalFeatureSettingsT +{ + /** + * Check whether the given experimental feature is enabled. + */ + bool isEnabled(const ExperimentalFeature &) const; + + /** + * Require an experimental feature be enabled, throwing an error if it is + * not. + */ + void require(const ExperimentalFeature &) const; + + /** + * `std::nullopt` pointer means no feature, which means there is nothing that could be + * disabled, and so the function returns true in that case. + */ + bool isEnabled(const std::optional &) const; + + /** + * `std::nullopt` pointer means no feature, which means there is nothing that could be + * disabled, and so the function does nothing in that case. + */ + void require(const std::optional &) const; +}; + +const extern ExperimentalFeatureSettings experimentalFeatureSettingsDefaults; + +// FIXME: don't use a global variable. +extern ExperimentalFeatureSettings experimentalFeatureSettings; + +} // namespace nix diff --git a/src/libutil/include/nix/util/experimental-features.hh b/src/libutil/include/nix/util/experimental-features.hh index 1eabc34619b..af4e06ad3d5 100644 --- a/src/libutil/include/nix/util/experimental-features.hh +++ b/src/libutil/include/nix/util/experimental-features.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "nix/util/config-abstract.hh" #include "nix/util/error.hh" #include "nix/util/types.hh" diff --git a/src/libutil/include/nix/util/fs-sink.hh b/src/libutil/include/nix/util/fs-sink.hh index f96fe3ef954..dcf1382a046 100644 --- a/src/libutil/include/nix/util/fs-sink.hh +++ b/src/libutil/include/nix/util/fs-sink.hh @@ -1,12 +1,19 @@ #pragma once ///@file +#include "nix/util/config-abstract.hh" #include "nix/util/serialise.hh" #include "nix/util/source-accessor.hh" #include "nix/util/file-system.hh" namespace nix { +template class R> +struct RestoreSinkSettings +{ + R preallocateContents; +}; + /** * Actions on an open regular file in the process of creating it. * @@ -22,6 +29,9 @@ struct CreateRegularFileSink : Sink virtual void preallocateContents(uint64_t size) {}; }; +// FIXME: don't use a global variable. +extern RestoreSinkSettings restoreSinkSettings; + struct FileSystemObjectSink { virtual ~FileSystemObjectSink() = default; diff --git a/src/libutil/include/nix/util/git.hh b/src/libutil/include/nix/util/git.hh index 5140c76c493..8db9902fa09 100644 --- a/src/libutil/include/nix/util/git.hh +++ b/src/libutil/include/nix/util/git.hh @@ -6,6 +6,7 @@ #include #include "nix/util/types.hh" +#include "nix/util/experimental-feature-settings.hh" #include "nix/util/serialise.hh" #include "nix/util/hash.hh" #include "nix/util/source-path.hh" diff --git a/src/libutil/include/nix/util/hash.hh b/src/libutil/include/nix/util/hash.hh index f4d137bd0ce..3007e75177b 100644 --- a/src/libutil/include/nix/util/hash.hh +++ b/src/libutil/include/nix/util/hash.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "nix/util/configuration.hh" +#include "nix/util/experimental-feature-settings.hh" #include "nix/util/types.hh" #include "nix/util/serialise.hh" #include "nix/util/file-system.hh" diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index 500d443e6e2..70dc83eee36 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -2,7 +2,7 @@ ///@file #include "nix/util/error.hh" -#include "nix/util/configuration.hh" +#include "nix/util/config-abstract.hh" #include "nix/util/file-descriptor.hh" #include "nix/util/finally.hh" @@ -43,31 +43,14 @@ typedef enum { typedef uint64_t ActivityId; -struct LoggerSettings : Config +template class R> +struct LoggerSettings { - Setting showTrace{ - this, - false, - "show-trace", - R"( - Whether Nix should print out a stack trace in case of Nix - expression evaluation errors. - )"}; - - Setting jsonLogPath{ - this, - "", - "json-log-path", - R"( - A file or unix socket to which JSON records of Nix's log output are - written, in the same format as `--log-format internal-json` - (without the `@nix ` prefixes on each line). - Concurrent writes to the same file by multiple Nix processes are not supported and - may result in interleaved or corrupted log records. - )"}; + R showTrace; + R jsonLogPath; }; -extern LoggerSettings loggerSettings; +extern LoggerSettings loggerSettings; class Logger { diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 91ad71ef0b4..aa726d15b50 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -3,11 +3,8 @@ include_dirs = [ include_directories('../..') ] headers = files( - 'abstract-setting-to-json.hh', 'ansicolor.hh', 'archive.hh', - 'args.hh', - 'args/root.hh', 'array-from-string-literal.hh', 'base-n.hh', 'base-nix-32.hh', @@ -20,9 +17,7 @@ headers = files( 'compression.hh', 'compute-levels.hh', 'config-abstract.hh', - 'config-global.hh', - 'config-impl.hh', - 'configuration.hh', + 'config-abstract.hh', 'current-process.hh', 'english.hh', 'environment-variables.hh', diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 997110617b3..28629f95365 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -3,7 +3,6 @@ #include "nix/util/environment-variables.hh" #include "nix/util/terminal.hh" #include "nix/util/util.hh" -#include "nix/util/config-global.hh" #include "nix/util/source-path.hh" #include "nix/util/position.hh" #include "nix/util/sync.hh" @@ -16,9 +15,15 @@ namespace nix { -LoggerSettings loggerSettings; +const extern LoggerSettings loggerSettingsDefaults = { + .showTrace = {.value = false}, + .jsonLogPath = + { + .value = "", + }, +}; -static GlobalConfig::Register rLoggerSettings(&loggerSettings); +LoggerSettings loggerSettings = loggerSettingsDefaults; static thread_local ActivityId curActivity = 0; diff --git a/src/libutil/meson.build b/src/libutil/meson.build index ffd1ebd49f2..49182d1558a 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -116,20 +116,18 @@ subdir('nix-meson-build-support/common') sources = [ config_priv_h ] + files( 'archive.cc', - 'args.cc', 'base-n.cc', 'base-nix-32.cc', 'canon-path.cc', 'compression.cc', 'compute-levels.cc', - 'config-global.cc', - 'configuration.cc', 'current-process.cc', 'english.cc', 'environment-variables.cc', 'error.cc', 'executable-path.cc', 'exit.cc', + 'experimental-feature-settings.cc', 'experimental-features.cc', 'file-content-address.cc', 'file-descriptor.cc', diff --git a/src/nix/config.cc b/src/nix/config.cc index c2a9fd8e2fe..0821b3d1966 100644 --- a/src/nix/config.cc +++ b/src/nix/config.cc @@ -2,7 +2,7 @@ #include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/store/store-api.hh" -#include "nix/util/config-global.hh" +#include "nix/main/config-global.hh" #include diff --git a/src/nix/develop.cc b/src/nix/develop.cc index d3381a9885a..1089f375bdf 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -1,4 +1,4 @@ -#include "nix/util/config-global.hh" +#include "nix/main/config-global.hh" #include "nix/expr/eval.hh" #include "nix/cmd/installable-flake.hh" #include "nix/cmd/command-installable-value.hh" diff --git a/src/nix/main.cc b/src/nix/main.cc index e3c081770c9..82cf7167a80 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -1,4 +1,4 @@ -#include "nix/util/args/root.hh" +#include "nix/main/args/root.hh" #include "nix/util/current-process.hh" #include "nix/cmd/command.hh" #include "nix/main/common-args.hh" diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 5dd53e9328b..526443b299f 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,6 +1,6 @@ #include "nix/expr/eval.hh" #include "nix/expr/eval-settings.hh" -#include "nix/util/config-global.hh" +#include "nix/main/config-global.hh" #include "nix/store/globals.hh" #include "nix/store/store-open.hh" #include "nix/cmd/command.hh" diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 4306e73ebc8..6a3aefb22ca 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -11,7 +11,7 @@ #include "nix/util/serialise.hh" #include "nix/util/archive.hh" #include "nix/store/globals.hh" -#include "nix/util/config-global.hh" +#include "nix/main/config-global.hh" #include "nix/store/derivations.hh" #include "nix/util/finally.hh" #include "nix/cmd/legacy.hh" diff --git a/tests/functional/plugins/plugintest.cc b/tests/functional/plugins/plugintest.cc index e8f80a4aa96..102eab95618 100644 --- a/tests/functional/plugins/plugintest.cc +++ b/tests/functional/plugins/plugintest.cc @@ -1,4 +1,4 @@ -#include "nix/util/config-global.hh" +#include "nix/main/config-global.hh" #include "nix/expr/primops.hh" using namespace nix;