Test Lazy trees with integration tests

src/libcmd/installable-value.cc

@@ -57,7 +57,8 @@ std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths
     else if (v.type() == nString) {
         return {{
             .path = DerivedPath::fromSingle(
-                state->coerceToSingleDerivedPath(pos, v, errorCtx)),
+                state->devirtualize(
+                    state->coerceToSingleDerivedPath(pos, v, errorCtx))),
             .info = make_ref<ExtraPathInfo>(),
         }};
     }

src/libexpr-test-support/include/nix/expr/tests/value/context.hh

@@ -23,6 +23,11 @@ struct Arbitrary<NixStringContextElem::DrvDeep> {
     static Gen<NixStringContextElem::DrvDeep> arbitrary();
 };
 
+template<>
+struct Arbitrary<NixStringContextElem::Path> {
+    static Gen<NixStringContextElem::Path> arbitrary();
+};
+
 template<>
 struct Arbitrary<NixStringContextElem> {
     static Gen<NixStringContextElem> arbitrary();

src/libexpr-test-support/tests/value/context.cc

@@ -15,6 +15,15 @@ Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arb
     });
 }
 
+Gen<NixStringContextElem::Path> Arbitrary<NixStringContextElem::Path>::arbitrary()
+{
+    return gen::map(gen::arbitrary<StorePath>(), [](StorePath storePath) {
+        return NixStringContextElem::Path{
+            .storePath = storePath,
+        };
+    });
+}
+
 Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
 {
     return gen::mapcat(
@@ -30,6 +39,9 @@ Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
             case 2:
                 return gen::map(
                     gen::arbitrary<NixStringContextElem::Built>(), [](NixStringContextElem a) { return a; });
+            case 3:
+                return gen::map(
+                    gen::arbitrary<NixStringContextElem::Path>(), [](NixStringContextElem a) { return a; });
             default:
                 assert(false);
             }

src/libexpr/eval-cache.cc

@@ -618,18 +618,21 @@ string_t AttrCursor::getStringWithContext()
             if (auto s = std::get_if<string_t>(&cachedValue->second)) {
                 bool valid = true;
                 for (auto & c : s->second) {
-                    const StorePath & path = std::visit(overloaded {
-                        [&](const NixStringContextElem::DrvDeep & d) -> const StorePath & {
-                            return d.drvPath;
+                    const StorePath * path = std::visit(overloaded {
+                        [&](const NixStringContextElem::DrvDeep & d) -> const StorePath * {
+                            return &d.drvPath;
                         },
-                        [&](const NixStringContextElem::Built & b) -> const StorePath & {
-                            return b.drvPath->getBaseStorePath();
+                        [&](const NixStringContextElem::Built & b) -> const StorePath * {
+                            return &b.drvPath->getBaseStorePath();
                         },
-                        [&](const NixStringContextElem::Opaque & o) -> const StorePath & {
-                            return o.path;
+                        [&](const NixStringContextElem::Opaque & o) -> const StorePath * {
+                            return &o.path;
+                        },
+                        [&](const NixStringContextElem::Path & p) -> const StorePath * {
+                            return nullptr;
                         },
                     }, c.raw);
-                    if (!root->state.store->isValidPath(path)) {
+                    if (!path || !root->state.store->isValidPath(*path)) {
                         valid = false;
                         break;
                     }

src/libexpr/eval.cc

@@ -15,6 +15,7 @@
 #include "nix/expr/print.hh"
 #include "nix/fetchers/filtering-source-accessor.hh"
 #include "nix/util/memory-source-accessor.hh"
+#include "nix/util/mounted-source-accessor.hh"
 #include "nix/expr/gc-small-vector.hh"
 #include "nix/util/url.hh"
 #include "nix/fetchers/fetch-to-store.hh"
@@ -269,7 +270,7 @@ EvalState::EvalState(
                    exception, and make union source accessor
                    catch it, so we don't need to do this hack.
                  */
-                {CanonPath(store->storeDir), store->getFSAccessor(settings.pureEval)},
+                {CanonPath(store->storeDir), makeFSSourceAccessor(dirOf(store->toRealPath(StorePath::dummy)))}
             }))
     , rootFS(
         ({
@@ -284,12 +285,9 @@ EvalState::EvalState(
                /nix/store while using a chroot store. */
             auto accessor = getFSSourceAccessor();
 
-            auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy));
-            if (settings.pureEval || store->storeDir != realStoreDir) {
-                accessor = settings.pureEval
-                    ? storeFS
-                    : makeUnionSourceAccessor({accessor, storeFS});
-            }
+            accessor = settings.pureEval
+                ? storeFS.cast<SourceAccessor>()
+                : makeUnionSourceAccessor({accessor, storeFS});
 
             /* Apply access control if needed. */
             if (settings.restrictEval || settings.pureEval)
@@ -972,7 +970,16 @@ void EvalState::mkPos(Value & v, PosIdx p)
     auto origin = positions.originOf(p);
     if (auto path = std::get_if<SourcePath>(&origin)) {
         auto attrs = buildBindings(3);
-        attrs.alloc(sFile).mkString(path->path.abs());
+        if (path->accessor == rootFS && store->isInStore(path->path.abs()))
+            // FIXME: only do this for virtual store paths?
+            attrs.alloc(sFile).mkString(path->path.abs(),
+                {
+                    NixStringContextElem::Path{
+                        .storePath = store->toStorePath(path->path.abs()).first
+                    }
+                });
+        else
+            attrs.alloc(sFile).mkString(path->path.abs());
         makePositionThunks(*this, p, attrs.alloc(sLine), attrs.alloc(sColumn));
         v.mkAttrs(attrs);
     } else
@@ -2098,7 +2105,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     else if (firstType == nFloat)
         v.mkFloat(nf);
     else if (firstType == nPath) {
-        if (!context.empty())
+        if (hasContext(context))
             state.error<EvalError>("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow();
         v.mkPath(state.rootPath(CanonPath(str())));
     } else
@@ -2297,7 +2304,10 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s
 {
     auto s = forceString(v, pos, errorCtx);
     if (v.context()) {
-        error<EvalError>("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow();
+        NixStringContext context;
+        copyContext(v, context);
+        if (hasContext(context))
+            error<EvalError>("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow();
     }
     return s;
 }
@@ -2346,14 +2356,26 @@ BackedStringView EvalState::coerceToString(
     }
 
     if (v.type() == nPath) {
+        // FIXME: instead of copying the path to the store, we could
+        // return a virtual store path that lazily copies the path to
+        // the store in devirtualize().
         return
             !canonicalizePath && !copyToStore
             ? // FIXME: hack to preserve path literals that end in a
               // slash, as in /foo/${x}.
               v.payload.path.path
             : copyToStore
             ? store->printStorePath(copyPathToStore(context, v.path()))
-            : std::string(v.path().path.abs());
+            : ({
+                auto path = v.path();
+                if (path.accessor == rootFS && store->isInStore(path.path.abs())) {
+                    context.insert(
+                        NixStringContextElem::Path{
+                            .storePath = store->toStorePath(path.path.abs()).first
+                        });
+                }
+                std::string(path.path.abs());
+              });
     }
 
     if (v.type() == nAttrs) {
@@ -2436,7 +2458,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
                 *store,
                 path.resolveSymlinks(SymlinkResolution::Ancestors),
                 settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
-                path.baseName(),
+                computeBaseName(path),
                 ContentAddressMethod::Raw::NixArchive,
                 nullptr,
                 repair);
@@ -2491,7 +2513,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon
     auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
     if (auto storePath = store->maybeParseStorePath(path))
         return *storePath;
-    error<EvalError>("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow();
+    error<EvalError>("cannot coerce '%s' to a store path because it is not a subpath of the Nix store", path).withTrace(pos, errorCtx).debugThrow();
 }
 
 
@@ -2517,6 +2539,11 @@ std::pair<SingleDerivedPath, std::string_view> EvalState::coerceToSingleDerivedP
         [&](NixStringContextElem::Built && b) -> SingleDerivedPath {
             return std::move(b);
         },
+        [&](NixStringContextElem::Path && p) -> SingleDerivedPath {
+            error<EvalError>(
+                "string '%s' has no context",
+                s).withTrace(pos, errorCtx).debugThrow();
+        },
     }, ((NixStringContextElem &&) *context.begin()).raw);
     return {
         std::move(derivedPath),
@@ -3100,6 +3127,11 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
 
         auto res = (r / CanonPath(suffix)).resolveSymlinks();
         if (res.pathExists()) return res;
+
+        // Backward compatibility hack: throw an exception if access
+        // to this path is not allowed.
+        if (auto accessor = res.accessor.dynamic_pointer_cast<FilteringSourceAccessor>())
+            accessor->checkAccess(res.path);
     }
 
     if (hasPrefix(path, "nix/"))
@@ -3170,6 +3202,11 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
         if (path.resolveSymlinks().pathExists())
             return finish(std::move(path));
         else {
+            // Backward compatibility hack: throw an exception if access
+            // to this path is not allowed.
+            if (auto accessor = path.accessor.dynamic_pointer_cast<FilteringSourceAccessor>())
+                accessor->checkAccess(path.path);
+
             logWarning({
                 .msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value)
             });

src/libexpr/include/nix/expr/eval-settings.hh

@@ -247,6 +247,11 @@ struct EvalSettings : Config
 
           This option can be enabled by setting `NIX_ABORT_ON_WARN=1` in the environment.
         )"};
+
+    Setting<bool> lazyTrees{this, true, "lazy-trees",
+        R"(
+          If set to true, flakes and trees fetched by [`builtins.fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) are only copied to the Nix store when they're used as a dependency of a derivation. This avoids copying (potentially large) source trees unnecessarily.
+        )"};
 };
 
 /**

src/libexpr/include/nix/expr/eval.hh

@@ -37,13 +37,15 @@ class Store;
 namespace fetchers {
 struct Settings;
 struct InputCache;
+struct Input;
 }
 struct EvalSettings;
 class EvalState;
 class StorePath;
 struct SingleDerivedPath;
 enum RepairFlag : bool;
 struct MemorySourceAccessor;
+struct MountedSourceAccessor;
 namespace eval_cache {
     class EvalCache;
 }
@@ -272,7 +274,7 @@ public:
     /**
      * The accessor corresponding to `store`.
      */
-    const ref<SourceAccessor> storeFS;
+    const ref<MountedSourceAccessor> storeFS;
 
     /**
      * The accessor for the root filesystem.
@@ -450,6 +452,15 @@ public:
 
     void checkURI(const std::string & uri);
 
+    /**
+     * Mount an input on the Nix store.
+     */
+    StorePath mountInput(
+        fetchers::Input & input,
+        const fetchers::Input & originalInput,
+        ref<SourceAccessor> accessor,
+        bool requireLockable);
+
     /**
      * Parse a Nix expression from the specified file.
      */
@@ -559,6 +570,18 @@ public:
     std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v,
         NixStringContext & context, bool coerceMore = false, bool copyToStore = true);
 
+    StorePath devirtualize(
+        const StorePath & path,
+        StringMap * rewrites = nullptr);
+
+    SingleDerivedPath devirtualize(
+        const SingleDerivedPath & path,
+        StringMap * rewrites = nullptr);
+
+    std::string devirtualize(
+        std::string_view s,
+        const NixStringContext & context);
+
     /**
      * String coercion.
      *
@@ -574,6 +597,19 @@ public:
 
     StorePath copyPathToStore(NixStringContext & context, const SourcePath & path);
 
+
+    /**
+     * Compute the base name for a `SourcePath`. For non-store paths,
+     * this is just `SourcePath::baseName()`. But for store paths, for
+     * backwards compatibility, it needs to be `<hash>-source`,
+     * i.e. as if the path were copied to the Nix store. This results
+     * in a "double-copied" store path like
+     * `/nix/store/<hash1>-<hash2>-source`. We don't need to
+     * materialize /nix/store/<hash2>-source though. Still, this
+     * requires reading/hashing the path twice.
+     */
+    std::string computeBaseName(const SourcePath & path);
+
     /**
      * Path coercion.
      *

src/libexpr/include/nix/expr/print-ambiguous.hh

@@ -15,10 +15,10 @@ namespace nix {
  * See: https://github.com/NixOS/nix/issues/9730
  */
 void printAmbiguous(
-    Value &v,
-    const SymbolTable &symbols,
-    std::ostream &str,
-    std::set<const void *> *seen,
+    EvalState & state,
+    Value & v,
+    std::ostream & str,
+    std::set<const void *> * seen,
     int depth);
 
 }

src/libexpr/include/nix/expr/value/context.hh

@@ -54,10 +54,35 @@ struct NixStringContextElem {
      */
     using Built = SingleDerivedPath::Built;
 
+    /**
+     * A store path that will not result in a store reference when
+     * used in a derivation or toFile.
+     *
+     * When you apply `builtins.toString` to a path value representing
+     * a path in the Nix store (as is the case with flake inputs),
+     * historically you got a string without context
+     * (e.g. `/nix/store/...-source`). This is broken, since it allows
+     * you to pass a store path to a derivation/toFile without a
+     * proper store reference. This is especially a problem with lazy
+     * trees, since the store path is a virtual path that doesn't
+     * exist.
+     *
+     * For backwards compatibility, and to warn users about this
+     * unsafe use of `toString`, we keep track of such strings as a
+     * special type of context.
+     */
+    struct Path
+    {
+        StorePath storePath;
+
+        GENERATE_CMP(Path, me->storePath);
+    };
+
     using Raw = std::variant<
         Opaque,
         DrvDeep,
-        Built
+        Built,
+        Path
     >;
 
     Raw raw;
@@ -82,4 +107,10 @@ struct NixStringContextElem {
 
 typedef std::set<NixStringContextElem> NixStringContext;
 
+/**
+ * Returns false if `context` has no elements other than
+ * `NixStringContextElem::Path`.
+ */
+bool hasContext(const NixStringContext & context);
+
 }