From 3a63fc6cd515fb009b17d864fede23a356832a5e Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Wed, 21 Oct 2020 21:31:19 +0200
Subject: [PATCH] Allow substituting paths when building remotely using
`ssh-ng://`
Until now, it was not possible to substitute missing paths from e.g.
`https://cache.nixos.org` on a remote server when building on it using
the new `ssh-ng` protocol.
This is because every store implementation except legacy `ssh://`
ignores the substitution flag passed to `Store::queryValidPaths` while
the `legacy-ssh-store` substitutes the remote store using
`cmdQueryValidPaths` when the remote store is opened with `nix-store
--serve`.
This patch slightly modifies the daemon protocol to allow passing an
integer value suggesting whether to substitute missing paths during
`wopQueryValidPaths`. To implement this on the daemon-side, the
substitution logic from `nix-store --serve` has been moved into a
protected method named `Store::substitutePaths` which gets currently
called from `LocalStore::queryValidPaths` and `Store::queryValidPaths`
if `maybeSubstitute` is `true`.
Fixes #2770
---
flake.nix | 3 ++-
src/libstore/daemon.cc | 11 ++++++++++-
src/libstore/remote-store.cc | 3 +++
src/libstore/store-api.cc | 22 ++++++++++++++++++++++
src/libstore/store-api.hh | 5 +++++
src/libstore/worker-protocol.hh | 2 +-
src/nix-store/nix-store.cc | 23 +----------------------
7 files changed, 44 insertions(+), 25 deletions(-)
diff --git a/flake.nix b/flake.nix
index 2abbdff53..8ff048be3 100644
--- a/flake.nix
+++ b/flake.nix
@@ -12,7 +12,7 @@
versionSuffix =
if officialRelease
then ""
- else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified)}_${self.shortRev or "dirty"}";
+ else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
officialRelease = false;
@@ -117,6 +117,7 @@
nix = with final; with commonDeps pkgs; (stdenv.mkDerivation {
name = "nix-${version}";
+ inherit version;
src = self;
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 4dbc7ba38..60cca4fda 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -274,8 +274,17 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopQueryValidPaths: {
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+
+ SubstituteFlag substitute = NoSubstitute;
+ if (GET_PROTOCOL_MINOR(clientVersion) >= 27) {
+ substitute = readInt(from) ? Substitute : NoSubstitute;
+ }
+
logger->startWork();
- auto res = store->queryValidPaths(paths);
+ if (substitute) {
+ store->substitutePaths(paths);
+ }
+ auto res = store->queryValidPaths(paths, substitute);
logger->stopWork();
worker_proto::write(*store, to, res);
break;
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index b6f70057d..c27a0f278 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -259,6 +259,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
} else {
conn->to << wopQueryValidPaths;
worker_proto::write(*this, conn->to, paths);
+ if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
+ conn->to << (settings.buildersUseSubstitutes ? 1 : 0);
+ }
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
}
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 83d3a1fa1..3129d6637 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -522,6 +522,28 @@ void Store::queryPathInfo(const StorePath & storePath,
}
+void Store::substitutePaths(const StorePathSet & paths)
+{
+ std::vector<StorePathWithOutputs> paths2;
+ for (auto & path : paths)
+ if (!path.isDerivation())
+ paths2.push_back({path});
+ uint64_t downloadSize, narSize;
+ StorePathSet willBuild, willSubstitute, unknown;
+ queryMissing(paths2,
+ willBuild, willSubstitute, unknown, downloadSize, narSize);
+
+ if (!willSubstitute.empty())
+ try {
+ std::vector<StorePathWithOutputs> subs;
+ for (auto & p : willSubstitute) subs.push_back({p});
+ buildPaths(subs);
+ } catch (Error & e) {
+ logWarning(e.info());
+ }
+}
+
+
StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute)
{
struct State
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index f77bc21d1..9a373b561 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -360,6 +360,11 @@ protected:
public:
+ /* If requested, substitute missing paths. This
+ implements nix-copy-closure's --use-substitutes
+ flag. */
+ void substitutePaths(const StorePathSet & paths);
+
/* Query which of the given paths is valid. Optionally, try to
substitute missing paths. */
virtual StorePathSet queryValidPaths(const StorePathSet & paths,
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index b3705578e..63bd6ea49 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
-#define PROTOCOL_VERSION 0x11a
+#define PROTOCOL_VERSION 0x11b
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 6c2702bfe..54394e921 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -830,29 +830,8 @@ static void opServe(Strings opFlags, Strings opArgs)
for (auto & path : paths)
store->addTempRoot(path);
- /* If requested, substitute missing paths. This
- implements nix-copy-closure's --use-substitutes
- flag. */
if (substitute && writeAllowed) {
- /* Filter out .drv files (we don't want to build anything). */
- std::vector<StorePathWithOutputs> paths2;
- for (auto & path : paths)
- if (!path.isDerivation())
- paths2.push_back({path});
- uint64_t downloadSize, narSize;
- StorePathSet willBuild, willSubstitute, unknown;
- store->queryMissing(paths2,
- willBuild, willSubstitute, unknown, downloadSize, narSize);
- /* FIXME: should use ensurePath(), but it only
- does one path at a time. */
- if (!willSubstitute.empty())
- try {
- std::vector<StorePathWithOutputs> subs;
- for (auto & p : willSubstitute) subs.push_back({p});
- store->buildPaths(subs);
- } catch (Error & e) {
- logWarning(e.info());
- }
+ store->substitutePaths(paths);
}
worker_proto::write(*store, out, store->queryValidPaths(paths));
--
GitLab