#include "nix/util/archive.hh" #include "nix/store/binary-cache-store.hh" #include "nix/util/compression.hh" #include "nix/store/derivations.hh" #include "nix/util/source-accessor.hh" #include "nix/store/nar-info-disk-cache.hh" #include "nix/store/nar-info.hh" #include "nix/util/sync.hh" #include "nix/store/remote-fs-accessor.hh" #include "nix/util/nar-accessor.hh" #include "nix/util/thread-pool.hh" #include "nix/util/callback.hh" #include "nix/util/signals.hh" #include "nix/util/archive.hh" #include "nix/util/util.hh" #include #include #include #include #include #include #include namespace nix { void BinaryCacheStoreConfig::anchor() {} void BinaryCacheStore::anchor() {} void NoSuchBinaryCacheFile::anchor() {} BinaryCacheStore::BinaryCacheStore(Config & config) : config{config} { if (auto & skf = config.secretKeyFile.get()) signers.push_back(std::make_unique(SecretKey{readFile(*skf)})); if (config.secretKeyFiles != "") { std::stringstream ss(config.secretKeyFiles); std::string keyPath; while (std::getline(ss, keyPath, ',')) { signers.push_back(std::make_unique(SecretKey{readFile(keyPath)})); } } StringSink sink; sink << narVersionMagic1; narMagic = sink.s; } void BinaryCacheStore::init() { auto cacheInfo = getNixCacheInfo(); if (!cacheInfo) { upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); } else { for (auto & line : tokenizeString(*cacheInfo, "\n")) { size_t colon = line.find(':'); if (colon == std::string::npos) continue; auto name = line.substr(0, colon); auto value = trim(line.substr(colon + 1, std::string::npos)); if (name == "StoreDir") { if (value != storeDir) throw Error( "binary cache '%s' is for Nix stores with prefix '%s', not '%s'", config.getHumanReadableURI(), value, storeDir); } else if (name == "WantMassQuery") { config.wantMassQuery.setDefault(value == "1"); } else if (name == "Priority") { config.priority.setDefault(std::stoi(value)); } } } } std::optional BinaryCacheStore::getNixCacheInfo() { return getFile(cacheInfoFile); } void BinaryCacheStore::upsertFile( const std::string & path, std::string && data, const std::string & mimeType, uint64_t sizeHint) { StringSource source{data}; upsertFile(path, source, mimeType, sizeHint); } void BinaryCacheStore::getFile(const std::string & path, Callback> callback) noexcept { try { callback(getFile(path)); } catch (...) { callback.rethrow(); } } void BinaryCacheStore::getFile(const std::string & path, Sink & sink) { std::promise> promise; getFile(path, {[&](std::future> result) { try { promise.set_value(result.get()); } catch (...) { promise.set_exception(std::current_exception()); } }}); sink(*promise.get_future().get()); } std::optional BinaryCacheStore::getFile(const std::string & path) { StringSink sink; try { getFile(path, sink); } catch (NoSuchBinaryCacheFile &) { return std::nullopt; } return std::move(sink.s); } std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath) { return std::string(storePath.hashPart()) + ".narinfo"; } void BinaryCacheStore::writeNarInfo(ref narInfo) { auto narInfoFile = narInfoFileFor(narInfo->path); upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); if (pathInfoCache) pathInfoCache->lock()->upsert(narInfo->path, PathInfoCacheValue{.value = std::shared_ptr(narInfo)}); if (diskCache) diskCache->upsertNarInfo( config.getReference().render(/*FIXME withParams=*/false), std::string(narInfo->path.hashPart()), std::shared_ptr(narInfo)); } ref BinaryCacheStore::uploadData(Source & narSource, RepairFlag repair, fun mkInfo) { auto fdTemp = createAnonymousTempFile(); auto now1 = std::chrono::steady_clock::now(); /* Read the NAR simultaneously into a CompressionSink+FileSink (to write the compressed NAR to disk), into a HashSink (to get the NAR hash), and into a NarAccessor (to get the NAR listing). */ HashSink fileHashSink{HashAlgorithm::SHA256}; std::shared_ptr narAccessor; HashSink narHashSink{HashAlgorithm::SHA256}; { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed{fileSink, fileHashSink}; bool parallel = config.parallelCompression.overridden ? config.parallelCompression.get() : config.compression.get() == CompressionAlgo::zstd; auto compressionSink = makeCompressionSink(config.compression, teeSinkCompressed, parallel, config.compressionLevel); TeeSink teeSinkUncompressed{*compressionSink, narHashSink}; TeeSource teeSource{narSource, teeSinkUncompressed}; narAccessor = makeNarAccessor(parseNarListing(teeSource)); compressionSink->finish(); fileSink.flush(); } auto now2 = std::chrono::steady_clock::now(); auto info = mkInfo(narHashSink.finish()); auto narInfo = make_ref(info); narInfo->compression = config.compression; auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar" + (config.compression == CompressionAlgo::xz ? ".xz" : config.compression == CompressionAlgo::bzip2 ? ".bz2" : config.compression == CompressionAlgo::zstd ? ".zst" : config.compression == CompressionAlgo::lzip ? ".lzip" : config.compression == CompressionAlgo::lz4 ? ".lz4" : config.compression == CompressionAlgo::brotli ? ".br" : ""); auto duration = std::chrono::duration_cast(now2 - now1).count(); printMsg( lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", printStorePath(narInfo->path), info.narSize, ((1.0 - (double) fileSize / info.narSize) * 100.0), duration); /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (config.writeNARListing) { nlohmann::json j = { {"version", 1}, {"root", narAccessor->getListing()}, }; upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); } /* Optionally maintain an index of DWARF debug info files consisting of JSON files named 'debuginfo/' that specify the NAR file and member containing the debug info. */ if (config.writeDebugInfo) { CanonPath buildIdDir("lib/debug/.build-id"); if (auto st = narAccessor->maybeLstat(buildIdDir); st && st->type == SourceAccessor::tDirectory) { ThreadPool threadPool(25); auto doFile = [&](std::string member, std::string key, std::string target) { checkInterrupt(); nlohmann::json json; json["archive"] = target; json["member"] = member; // FIXME: or should we overwrite? The previous link may point // to a GC'ed file, so overwriting might be useful... if (fileExists(key)) return; printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target); upsertFile(key, json.dump(), "application/json"); }; std::regex regex1("^[0-9a-f]{2}$"); std::regex regex2("^[0-9a-f]{38}\\.debug$"); for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) { auto dir = buildIdDir / s1; if (narAccessor->lstat(dir).type != SourceAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; for (auto & [s2, _type] : narAccessor->readDirectory(dir)) { auto debugPath = dir / s2; if (narAccessor->lstat(debugPath).type != SourceAccessor::tRegular || !std::regex_match(s2, regex2)) continue; auto buildId = s1 + s2; std::string key = "debuginfo/" + buildId; std::string target = "../" + narInfo->url; threadPool.enqueue(std::bind(doFile, std::string(debugPath.rel()), key, target)); } } threadPool.process(); } } /* Atomically write the NAR file. */ if (repair || !fileExists(narInfo->url)) { FdSource source{fdTemp.get()}; source.restart(); /* Seek back to the start of the file. */ upsertFile(narInfo->url, source, "application/x-nix-nar", narInfo->fileSize); } return narInfo; } void BinaryCacheStore::uploadNarInfo(ref narInfo) { /* Verify that all references are valid. This may do some .narinfo reads, but typically they'll already be cached. */ for (auto & ref : narInfo->references) try { if (ref != narInfo->path) queryPathInfo(ref); } catch (InvalidPath &) { throw Error( "cannot add '%s' to the binary cache because the reference '%s' is not valid", printStorePath(narInfo->path), printStorePath(ref)); } narInfo->sign(*this, signers); /* Atomically write the NAR info file.*/ writeNarInfo(narInfo); } ref BinaryCacheStore::addToStoreCommon( Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, fun mkInfo) { auto narInfo = uploadData(narSource, repair, std::move(mkInfo)); uploadNarInfo(narInfo); return narInfo; } void BinaryCacheStore::addToStore( const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs) { if (!repair && isValidPath(info.path)) return; addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) { /* FIXME reinstate these, once we can correctly do masked hash sink as needed. We need to throw here in case we uploaded a corrupted store path. */ // assert(info.narHash == nar.first); // assert(info.narSize == nar.second); return info; }}); } void BinaryCacheStore::addMultipleToStore( PathsSource && pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) { /* Index the paths to copy by store path so the graph nodes below can look up each path's info (NAR size, references) and source. */ std::map> *> infosMap; uint64_t bytesExpected = 0; for (auto & item : pathsToCopy) { bytesExpected += item.first.narSize; infosMap.insert_or_assign(item.first.path, &item); } act.setExpected(actCopyPath, bytesExpected); std::atomic nrDone{0}; std::atomic nrRunning{0}; auto showProgress = [&, nrTotal = pathsToCopy.size()]() { act.progress(nrDone, nrTotal, nrRunning); }; /* The NarInfos produced by uploading the NARs, to be consumed when writing the .narinfo files. Populated by the `UploadNar` nodes and read by the corresponding `UploadNarInfo` nodes. */ Sync>> narInfos_; /* The work graph has two kinds of nodes: uploading the NAR for a path (which has no dependencies, since NARs are independent of each other), and uploading the .narinfo for a path (which depends on the corresponding NAR upload and on the .narinfo uploads of all the path's references). Processing the latter in topological order maintains the closure invariant: whenever a .narinfo exists, the .narinfo files of all its references exist as well. */ struct UploadNar { StorePath path; uint64_t narSize; /* Order NAR uploads by descending size so that the largest (and typically slowest) NARs are started first. */ bool operator<(const UploadNar & other) const { return narSize != other.narSize ? narSize > other.narSize : path < other.path; } }; struct UploadNarInfo { StorePath path; bool operator<(const UploadNarInfo & other) const { return path < other.path; } }; /* `std::variant`'s `operator<` orders by alternative index first, so all `UploadNar` nodes sort (and thus get enqueued) before any `UploadNarInfo` node. TODO: uploading the debug info and NAR listings could be turned into separate graph nodes as well. */ using Node = std::variant; std::set nodes; for (auto & [path, item] : infosMap) { nodes.insert(UploadNar{path, item->first.narSize}); nodes.insert(UploadNarInfo{path}); } processGraph( nodes, [&](const Node & node) -> std::set { return std::visit( overloaded{ [&](const UploadNar &) -> std::set { /* NAR uploads have no dependencies. */ return {}; }, [&](const UploadNarInfo & n) -> std::set { std::set edges; auto & info = infosMap.at(n.path)->first; /* Wait for our own NAR to be uploaded ... */ edges.insert(UploadNar{n.path, info.narSize}); /* ... and for the .narinfo files of all references that are part of this copy (other references are already valid in the store). */ for (auto & ref : info.references) { if (ref != n.path && infosMap.count(ref)) edges.insert(UploadNarInfo{ref}); } return edges; }, }, node); }, [&](const Node & node) { checkInterrupt(); std::visit( overloaded{ [&](const UploadNar & n) { auto & [info, source_] = *infosMap.at(n.path); /* Make sure the Source object is destroyed when we're done, e.g. to release the connection lock held by LegacySSHStore::narFromPath(). */ auto source = std::move(source_); if (repair || !isValidPath(info.path)) { MaintainCount mc(nrRunning); showProgress(); auto narInfo = uploadData(*source, repair, [&](HashResult nar) { auto info2 = info; info2.ultimate = false; return info2; }); narInfos_.lock()->insert_or_assign(info.path, narInfo); } nrDone++; showProgress(); }, [&](const UploadNarInfo & n) { auto & info = infosMap.at(n.path)->first; if (!repair && isValidPath(info.path)) return; auto narInfo = narInfos_.lock()->at(n.path); uploadNarInfo(narInfo); }, }, node); }); } StorePath BinaryCacheStore::addToStoreFromDump( Source & dump, std::string_view name, FileSerialisationMethod dumpMethod, ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, RepairFlag repair) { std::optional caHash; std::string nar; // Calculating Git hash from NAR stream not yet implemented. May not // be possible to implement in single-pass if the NAR is in an // inconvenient order. Could fetch after uploading, however. if (hashMethod.getFileIngestionMethod() == FileIngestionMethod::Git) unsupported("addToStoreFromDump"); if (auto * dump2p = dynamic_cast(&dump)) { auto & dump2 = *dump2p; // Hack, this gives us a "replayable" source so we can compute // multiple hashes more easily. // // Only calculate if the dump is in the right format, however. if (static_cast(dumpMethod) == hashMethod.getFileIngestionMethod()) caHash = hashString(HashAlgorithm::SHA256, dump2.s); switch (dumpMethod) { case FileSerialisationMethod::NixArchive: // The dump is already NAR in this case, just use it. nar = dump2.s; break; case FileSerialisationMethod::Flat: { // The dump is Flat, so we need to convert it to NAR with a // single file. StringSink s; dumpString(dump2.s, s); nar = std::move(s.s); break; } } } else { // Otherwise, we have to do th same hashing as NAR so our single // hash will suffice for both purposes. if (dumpMethod != FileSerialisationMethod::NixArchive || hashAlgo != HashAlgorithm::SHA256) unsupported("addToStoreFromDump"); } StringSource narDump{nar}; // Use `narDump` if we wrote to `nar`. Source & narDump2 = nar.size() > 0 ? static_cast(narDump) : dump; return addToStoreCommon( narDump2, repair, CheckSigs, [&](HashResult nar) { auto info = ValidPathInfo::makeFromCA( *this, name, ContentAddressWithReferences::fromParts( hashMethod, caHash ? *caHash : nar.hash, { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed // without modulus .self = false, }), nar.hash); info.narSize = nar.numBytesDigested; return info; }) ->path; } bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) { // FIXME: this only checks whether a .narinfo with a matching hash // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even // though they shouldn't. Not easily fixed. return fileExists(narInfoFileFor(storePath)); } std::optional BinaryCacheStore::queryPathFromHashPart(const std::string & hashPart) { auto pseudoPath = StorePath(hashPart + "-" + MissingName); try { auto info = queryPathInfo(pseudoPath); return info->path; } catch (InvalidPath &) { return std::nullopt; } } void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) { auto info = queryPathInfo(storePath).cast(); /* makeDecompressionSink used to treat empty strings as "none". It seems impossible that it would actually end up here with an empty string though (since an empty `Compression: ' is treated as bzip2 when parsed from a .narinfo file and the narinfo disk cache wouldn't handle empty strings). TODO: Revisit this and convert to an assert probably or even made compression a non-optional field. */ auto decompressor = makeDecompressionSink(info->compression.value_or(CompressionAlgo::none), sink); try { getFile(info->url, *decompressor); } catch (NoSuchBinaryCacheFile & e) { throw SubstituteGone(std::move(e.info())); } decompressor->finish(); } void BinaryCacheStore::queryPathInfoUncached( const StorePath & storePath, Callback> callback) noexcept { auto callbackPtr = std::make_shared(std::move(callback)); try { auto uri = config.getReference().render(/*FIXME withParams=*/false); auto storePathS = printStorePath(storePath); auto act = std::make_shared( *logger, lvlTalkative, actQueryPathInfo, fmt("querying info about '%s' on '%s'", storePathS, uri), std::to_array({storePathS, uri})); PushActivity pact(act->id); auto narInfoFile = narInfoFileFor(storePath); getFile(narInfoFile, {[=, this](std::future> fut) { try { auto data = fut.get(); if (!data) return (*callbackPtr)({}); (*callbackPtr)( (std::shared_ptr) std::make_shared(*this, *data, narInfoFile)); (void) act; // force Activity into this lambda to ensure it stays alive } catch (...) { callbackPtr->rethrow(); } }}); } catch (...) { callbackPtr->rethrow(); } } StorePath BinaryCacheStore::addToStore( std::string_view name, const SourcePath & path, ContentAddressMethod method, HashAlgorithm hashAlgo, const StorePathSet & references, PathFilter & filter, RepairFlag repair) { /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default implementation of this method in terms of addToStoreFromDump. */ auto h = hashPath(path, method.getFileIngestionMethod(), hashAlgo, filter).first; auto source = sinkToSource([&](Sink & sink) { path.dumpPath(sink, filter); }); return addToStoreCommon( *source, repair, CheckSigs, [&](HashResult nar) { auto info = ValidPathInfo::makeFromCA( *this, name, ContentAddressWithReferences::fromParts( method, h, { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed // without modulus .self = false, }), nar.hash); info.narSize = nar.numBytesDigested; return info; }) ->path; } std::string BinaryCacheStore::makeRealisationPath(const DrvOutput & id) { return realisationsPrefix + "/" + id.drvPath.to_string() + "/" + id.outputName + ".doi"; } void BinaryCacheStore::queryRealisationUncached( const DrvOutput & id, Callback> callback) noexcept { auto outputInfoFilePath = makeRealisationPath(id); auto callbackPtr = std::make_shared(std::move(callback)); Callback> newCallback = {[=](std::future> fut) { try { auto data = fut.get(); if (!data) return (*callbackPtr)({}); std::shared_ptr realisation; try { realisation = std::make_shared(nlohmann::json::parse(*data)); } catch (Error & e) { e.addTrace( {}, "while parsing file '%s' as a build trace value for key '%s'", outputInfoFilePath, id.to_string()); throw; } return (*callbackPtr)(std::move(realisation)); } catch (...) { callbackPtr->rethrow(); } }}; getFile(outputInfoFilePath, std::move(newCallback)); } void BinaryCacheStore::registerDrvOutputUnchecked(const Realisation & info) { if (diskCache) diskCache->upsertRealisation(config.getReference().render(/*FIXME withParams=*/false), info); upsertFile( makeRealisationPath(info.id), static_cast(static_cast(info)).dump(), "application/json"); } ref BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath) { return make_ref(ref(shared_from_this()), requireValidPath, config.localNarCache); } ref BinaryCacheStore::getFSAccessor(bool requireValidPath) { return getRemoteFSAccessor(requireValidPath); } std::shared_ptr BinaryCacheStore::getFSAccessor(const StorePath & storePath, bool requireValidPath) { return getRemoteFSAccessor(requireValidPath)->accessObject(storePath); } void BinaryCacheStore::addSignatures(const StorePath & storePath, const std::set & sigs) { /* Note: this is inherently racy since there is no locking on binary caches. In particular, with S3 this unreliable, even when addSignatures() is called sequentially on a path, because S3 might return an outdated cached version. */ auto narInfo = make_ref((NarInfo &) *queryPathInfo(storePath)); narInfo->sigs.insert(sigs.begin(), sigs.end()); writeNarInfo(narInfo); } std::optional BinaryCacheStore::getBuildLogExact(const StorePath & path) { auto logPath = "log/" + std::string(baseNameOf(printStorePath(path))); debug("fetching build log from binary cache '%s/%s'", config.getHumanReadableURI(), logPath); return getFile(logPath); } void BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log) { assert(drvPath.isDerivation()); upsertFile( "log/" + std::string(drvPath.to_string()), (std::string) log, // FIXME: don't copy "text/plain; charset=utf-8"); } } // namespace nix