From a583a2bc59a4ee2b067e5520f6c5bc0c61852c32 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Wed, 7 Aug 2013 11:51:55 +0000
Subject: [PATCH] Run the daemon worker on the same CPU as the client
On a system with multiple CPUs, running Nix operations through the
daemon is significantly slower than "direct" mode:
$ NIX_REMOTE= nix-instantiate '<nixos>' -A system
real 0m0.974s
user 0m0.875s
sys 0m0.088s
$ NIX_REMOTE=daemon nix-instantiate '<nixos>' -A system
real 0m2.118s
user 0m1.463s
sys 0m0.218s
The main reason seems to be that the client and the worker get moved
to a different CPU after every call to the worker. This patch adds a
hack to lock them to the same CPU. With this, the overhead of going
through the daemon is very small:
$ NIX_REMOTE=daemon nix-instantiate '<nixos>' -A system
real 0m1.074s
user 0m0.809s
sys 0m0.098s
---
configure.ac | 4 +++
src/libstore/build.cc | 4 +++
src/libstore/local-store.cc | 2 ++
src/libstore/remote-store.cc | 13 +++++++-
src/libstore/worker-protocol.hh | 2 +-
src/libutil/Makefile.am | 4 +--
src/libutil/affinity.cc | 54 +++++++++++++++++++++++++++++++++
src/libutil/affinity.hh | 9 ++++++
src/nix-daemon/nix-daemon.cc | 4 +++
9 files changed, 92 insertions(+), 4 deletions(-)
create mode 100644 src/libutil/affinity.cc
create mode 100644 src/libutil/affinity.hh
diff --git a/configure.ac b/configure.ac
index 9ffefa914..89f009923 100644
--- a/configure.ac
+++ b/configure.ac
@@ -127,6 +127,10 @@ AC_CHECK_HEADERS([sys/mount.h], [], [],
AC_CHECK_FUNCS([lutimes])
+# Check for sched_setaffinity.
+AC_CHECK_FUNCS([sched_setaffinity])
+
+
# Check whether the store optimiser can optimise symlinks.
AC_MSG_CHECKING([whether it is possible to create a link to a symlink])
ln -s bla tmp_link
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1ca99dcc5..67e38da0b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -7,6 +7,7 @@
#include "local-store.hh"
#include "util.hh"
#include "archive.hh"
+#include "affinity.hh"
#include <map>
#include <sstream>
@@ -366,6 +367,8 @@ void Goal::trace(const format & f)
/* Common initialisation performed in child processes. */
static void commonChildInit(Pipe & logPipe)
{
+ restoreAffinity();
+
/* Put the child in a separate session (and thus a separate
process group) so that it has no controlling terminal (meaning
that e.g. ssh cannot open /dev/tty) and it doesn't receive
@@ -568,6 +571,7 @@ static void runSetuidHelper(const string & command,
args.push_back(0);
restoreSIGPIPE();
+ restoreAffinity();
execve(program.c_str(), (char * *) &args[0], 0);
throw SysError(format("executing `%1%'") % program);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 29095e1ea..34b4a5158 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -5,6 +5,7 @@
#include "pathlocks.hh"
#include "worker-protocol.hh"
#include "derivations.hh"
+#include "affinity.hh"
#include <iostream>
#include <algorithm>
@@ -1021,6 +1022,7 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
case 0: /* child */
try {
+ restoreAffinity();
if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
throw SysError("dupping stdin");
if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 2b5a93213..3764b4813 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -3,6 +3,7 @@
#include "remote-store.hh"
#include "worker-protocol.hh"
#include "archive.hh"
+#include "affinity.hh"
#include "globals.hh"
#include <sys/types.h>
@@ -15,7 +16,6 @@
#include <unistd.h>
#include <cstring>
-
namespace nix {
@@ -71,8 +71,19 @@ void RemoteStore::openConnection(bool reserveSpace)
if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
throw Error("Nix daemon protocol version not supported");
writeInt(PROTOCOL_VERSION, to);
+
+ if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) {
+ int cpu = lockToCurrentCPU();
+ if (cpu != -1) {
+ writeInt(1, to);
+ writeInt(cpu, to);
+ } else
+ writeInt(0, to);
+ }
+
if (GET_PROTOCOL_MINOR(daemonVersion) >= 11)
writeInt(reserveSpace, to);
+
processStderr();
}
catch (Error & e) {
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 07f825b92..9317f89c3 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
-#define PROTOCOL_VERSION 0x10d
+#define PROTOCOL_VERSION 0x10e
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
diff --git a/src/libutil/Makefile.am b/src/libutil/Makefile.am
index fe896eec5..0c4073e66 100644
--- a/src/libutil/Makefile.am
+++ b/src/libutil/Makefile.am
@@ -1,12 +1,12 @@
pkglib_LTLIBRARIES = libutil.la
libutil_la_SOURCES = util.cc hash.cc serialise.cc \
- archive.cc xml-writer.cc
+ archive.cc xml-writer.cc affinity.cc
libutil_la_LIBADD = ../boost/format/libformat.la
pkginclude_HEADERS = util.hh hash.hh serialise.hh \
- archive.hh xml-writer.hh types.hh
+ archive.hh xml-writer.hh types.hh affinity.hh
if !HAVE_OPENSSL
libutil_la_SOURCES += \
diff --git a/src/libutil/affinity.cc b/src/libutil/affinity.cc
new file mode 100644
index 000000000..3a20fd277
--- /dev/null
+++ b/src/libutil/affinity.cc
@@ -0,0 +1,54 @@
+#include "types.hh"
+#include "util.hh"
+#include "affinity.hh"
+
+#if HAVE_SCHED_H
+#include <sched.h>
+#endif
+
+namespace nix {
+
+
+static bool didSaveAffinity = false;
+static cpu_set_t savedAffinity;
+
+
+void setAffinityTo(int cpu)
+{
+#if HAVE_SCHED_SETAFFINITY
+ if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return;
+ didSaveAffinity = true;
+ printMsg(lvlDebug, format("locking this thread to CPU %1%") % cpu);
+ cpu_set_t newAffinity;
+ CPU_ZERO(&newAffinity);
+ CPU_SET(cpu, &newAffinity);
+ if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1)
+ printMsg(lvlError, format("failed to lock thread to CPU %1%") % cpu);
+#endif
+}
+
+
+int lockToCurrentCPU()
+{
+#if HAVE_SCHED_SETAFFINITY
+ if (getEnv("NIX_AFFINITY_HACK", "1") == "1") {
+ int cpu = sched_getcpu();
+ if (cpu != -1) setAffinityTo(cpu);
+ return cpu;
+ }
+#endif
+ return -1;
+}
+
+
+void restoreAffinity()
+{
+#if HAVE_SCHED_SETAFFINITY
+ if (!didSaveAffinity) return;
+ if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
+ printMsg(lvlError, "failed to restore affinity %1%");
+#endif
+}
+
+
+}
diff --git a/src/libutil/affinity.hh b/src/libutil/affinity.hh
new file mode 100644
index 000000000..c1bd28e13
--- /dev/null
+++ b/src/libutil/affinity.hh
@@ -0,0 +1,9 @@
+#pragma once
+
+namespace nix {
+
+void setAffinityTo(int cpu);
+int lockToCurrentCPU();
+void restoreAffinity();
+
+}
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index a6f29b7c9..86f5c0a24 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -4,6 +4,7 @@
#include "serialise.hh"
#include "worker-protocol.hh"
#include "archive.hh"
+#include "affinity.hh"
#include "globals.hh"
#include <cstring>
@@ -671,6 +672,9 @@ static void processConnection(bool trusted)
to.flush();
unsigned int clientVersion = readInt(from);
+ if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from))
+ setAffinityTo(readInt(from));
+
bool reserveSpace = true;
if (GET_PROTOCOL_MINOR(clientVersion) >= 11)
reserveSpace = readInt(from) != 0;
--
GitLab