From 4be4f6de56f4de77f6a376f1a40ed75eb641bb89 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 16 Nov 2016 16:21:30 +0100
Subject: [PATCH] S3BinaryCacheStore:: Eliminate a string copy while uploading
This cuts hydra-queue-runner's peak memory usage by about a third.
---
src/libstore/derivations.cc | 2 +-
src/libstore/s3-binary-cache-store.cc | 2 +-
src/libutil/hash.cc | 2 +-
src/libutil/util.hh | 14 ++++++++++++++
4 files changed, 17 insertions(+), 3 deletions(-)
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index d934bda38..5562d4689 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -155,7 +155,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
static Derivation parseDerivation(const string & s)
{
Derivation drv;
- std::istringstream str(s);
+ istringstream_nocopy str(s);
expect(str, "Derive([");
/* Parse the list of outputs. */
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index c11f2b06b..1bc8576a8 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -145,7 +145,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
.WithBucket(bucketName)
.WithKey(path);
- auto stream = std::make_shared<std::stringstream>(data);
+ auto stream = std::make_shared<istringstream_nocopy>(data);
request.SetBody(stream);
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 81aced0fd..49e781980 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -106,7 +106,7 @@ Hash parseHash(HashType ht, const string & s)
string s2(s, i * 2, 2);
if (!isxdigit(s2[0]) || !isxdigit(s2[1]))
throw BadHash(format("invalid hash ‘%1%’") % s);
- std::istringstream str(s2);
+ istringstream_nocopy str(s2);
int n;
str >> std::hex >> n;
hash.hash[i] = n;
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 50b96f7ed..a8f6f99b9 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -431,4 +431,18 @@ void callSuccess(
}
+/* A variant of std::istringstream that doesn't its string
+ argument. This is useful for large strings. The caller must ensure
+ that the string object is not destroyed while it's referenced by
+ this object. */
+struct istringstream_nocopy : public std::stringstream
+{
+ istringstream_nocopy(const std::string & s)
+ {
+ rdbuf()->pubsetbuf(
+ (char *) s.data(), s.size());
+ }
+};
+
+
}
--
GitLab