// Copyright Contributors to the DNF5 project. // Copyright Contributors to the libdnf project. // SPDX-License-Identifier: LGPL-2.1-or-later // // This file is part of libdnf: https://github.com/rpm-software-management/libdnf/ // // Libdnf is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 2.1 of the License, or // (at your option) any later version. // // Libdnf is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with libdnf. If not, see . #include "repo_downloader.hpp" #include "repo_pgp.hpp" #include "utils/fs/utils.hpp" #include "utils/string.hpp" #include "libdnf5/conf/const.hpp" #include "libdnf5/repo/repo.hpp" #include "libdnf5/repo/repo_errors.hpp" #include "libdnf5/utils/bgettext/bgettext-mark-domain.h" #include #include #include #include #include #include #include #include #include #define RECOGNIZED_CHKSUMS {"sha512", "sha256"} namespace libdnf5::repo { static void str_vector_to_char_array(const std::vector & vec, const char * arr[]) { for (size_t i = 0; i < vec.size(); ++i) { arr[i] = vec[i].c_str(); } arr[vec.size()] = nullptr; } static LrYumRepo * get_yum_repo(LibrepoResult & result) { LrYumRepo * yum_repo; result.get_info(LRR_YUM_REPO, &yum_repo); return yum_repo; } static LrYumRepoMd * get_yum_repomd(LibrepoResult & result) { LrYumRepoMd * yum_repomd; result.get_info(LRR_YUM_REPOMD, &yum_repomd); return yum_repomd; } int RepoDownloader::end_cb_full_download(void * data, LrTransferStatus status, const char * msg) { libdnf_assert(data != nullptr, "data in callback must be set"); auto download_callback_data = static_cast(data); auto cb_status = static_cast(status); if (cb_status == DownloadCallbacks::TransferStatus::SUCCESSFUL && download_callback_data) { // move all downloaded object from tmpdir to destdir for (auto & dir : std::filesystem::directory_iterator(download_callback_data->temp_download_target->get_path())) { auto tmp_item = dir.path(); auto target_item = download_callback_data->destination / tmp_item.filename(); std::filesystem::remove_all(target_item); utils::fs::move_recursive(tmp_item, target_item); } if (download_callback_data->load_repo) { download_callback_data->load_repo(download_callback_data->repo.get(), false); } } if (auto * download_callbacks = download_callback_data->repo->get_base()->get_download_callbacks()) { if (cb_status == DownloadCallbacks::TransferStatus::ERROR && download_callback_data->suppress_keyring_errors && is_signing_key_not_found_error(msg)) { return download_callbacks->end( download_callback_data->user_cb_data, DownloadCallbacks::TransferStatus::SUCCESSFUL, nullptr); } return download_callbacks->end(download_callback_data->user_cb_data, cb_status, msg); } return 0; } int RepoDownloader::end_cb_sync_check(void * data, LrTransferStatus status, const char * msg) { libdnf_assert(data != nullptr, "data in callback must be set"); auto cbd = static_cast(data); auto cb_status = static_cast(status); if (cb_status == DownloadCallbacks::TransferStatus::SUCCESSFUL && cbd) { cbd->is_in_sync = false; auto & repo = *cbd->repo.get(); auto & config = cbd->repo->get_config(); auto & download_data = cbd->repo->get_download_data(); if (!config.get_metalink_option().empty() && !config.get_metalink_option().get_value().empty()) { LrMetalink * metalink; download_data.handle->get_info(LRI_METALINK, &metalink); cbd->is_in_sync = RepoDownloader::is_metalink_in_sync(repo, metalink); } else { std::filesystem::path repomd; if (cbd->lr_target->repo) { repomd = cbd->lr_target->repo->repomd; } cbd->is_in_sync = RepoDownloader::is_repomd_in_sync(repo, repomd); if (!cbd->is_in_sync) { // We have already downloaded the new repomd, when we will go download the // rest of the metadata do just an update. if (std::filesystem::exists(repomd)) { download_data.handle->set_opt(LRO_UPDATE, 1); } } } if (cbd->load_repo && cbd->is_in_sync) { auto logger = repo.get_base()->get_logger(); logger->debug( "Using cache for repo \"{}\". It is expired, but matches the original.", repo.get_config().get_id()); cbd->load_repo(cbd->repo.get(), true); } } if (auto * download_callbacks = cbd->repo->get_base()->get_download_callbacks()) { if (cb_status == DownloadCallbacks::TransferStatus::ERROR && cbd->suppress_keyring_errors && is_signing_key_not_found_error(msg)) { return download_callbacks->end(cbd->user_cb_data, DownloadCallbacks::TransferStatus::SUCCESSFUL, nullptr); } return download_callbacks->end(cbd->user_cb_data, cb_status, msg); } return 0; } int RepoDownloader::progress_cb(void * data, double total_to_download, double downloaded) { if (!data) { return 0; } auto download_callback_data = static_cast(data); if (auto * download_callbacks = download_callback_data->repo->get_base()->get_download_callbacks()) { // "total_to_download" and "downloaded" from librepo are related to the currently downloaded file. // We add the size of previously downloaded files. // ignore zero progress events at the beginning of the download, so we don't start with 100% progress if (total_to_download != 0 || download_callback_data->prev_total_to_download != 0) { if (total_to_download != download_callback_data->prev_total_to_download) { download_callback_data->prev_total_to_download = total_to_download; download_callback_data->sum_prev_downloaded += download_callback_data->prev_downloaded; } download_callback_data->prev_downloaded = downloaded; } total_to_download += download_callback_data->sum_prev_downloaded; downloaded += download_callback_data->sum_prev_downloaded; return download_callbacks->progress(download_callback_data->user_cb_data, total_to_download, downloaded); } return 0; } void RepoDownloader::fastest_mirror_cb(void * data, LrFastestMirrorStages stage, void * ptr) { if (!data) { return; } auto download_callback_data = static_cast(data); if (auto * download_callbacks = download_callback_data->repo->get_base()->get_download_callbacks()) { const char * msg; std::string msg_string; if (ptr) { switch (stage) { case LR_FMSTAGE_CACHELOADING: case LR_FMSTAGE_CACHELOADINGSTATUS: case LR_FMSTAGE_STATUS: msg = static_cast(ptr); break; case LR_FMSTAGE_DETECTION: msg_string = std::to_string(*(static_cast(ptr))); msg = msg_string.c_str(); break; default: msg = nullptr; } } else { msg = nullptr; } download_callbacks->fastest_mirror( download_callback_data->user_cb_data, static_cast(stage), msg); } } int RepoDownloader::mirror_failure_cb(void * data, const char * msg, const char * url) { if (!data) { return 0; } auto download_callback_data = static_cast(data); if (auto * download_callbacks = download_callback_data->repo->get_base()->get_download_callbacks()) { // In this callback type we don't have the `metadata` type, pass NULL instead. The dnf5 callback accounts for this. return download_callbacks->mirror_failure(download_callback_data->user_cb_data, msg, url, NULL); } return 0; }; int RepoDownloader::mirror_failure_cb( void * data, const char * msg, const char * url, [[maybe_unused]] const char * metadata) { if (!data) { return 0; } auto download_callback_data = static_cast(data); if (auto * download_callbacks = download_callback_data->repo->get_base()->get_download_callbacks()) { return download_callbacks->mirror_failure(download_callback_data->user_cb_data, msg, url, metadata); } return 0; }; LibrepoError::LibrepoError(std::unique_ptr && lr_error) : Error(M_("Librepo error: {}"), std::string(lr_error->message)), code(lr_error->code) {} void RepoDownloader::add( libdnf5::repo::Repo & repo, const std::string & destdir, std::function load_repo) try { auto & cbd = callback_data.emplace_back(); cbd.repo = repo.get_weak_ptr(); cbd.destination = destdir; cbd.load_repo = load_repo; cbd.suppress_keyring_errors = suppress_keyring_errors; std::filesystem::create_directories(cbd.destination); cbd.temp_download_target = libdnf5::utils::fs::TempDir{cbd.destination, "tmpdir"}; auto & download_data = cbd.repo->get_download_data(); auto * download_callbacks = cbd.repo->get_base()->get_download_callbacks(); auto & config = cbd.repo->get_config(); if (download_callbacks) { cbd.user_cb_data = download_callbacks->add_new_download( download_data.user_data, !config.get_name_option().get_value().empty() ? config.get_name_option().get_value().c_str() : (!config.get_id().empty() ? config.get_id().c_str() : "unknown"), -1); cbd.prev_total_to_download = 0; cbd.prev_downloaded = 0; cbd.sum_prev_downloaded = 0; } download_data.handle = init_remote_handle(*cbd.repo, cbd.temp_download_target->get_path().c_str(), true); download_data.handle->set_opt(LRO_FASTESTMIRRORCB, static_cast(fastest_mirror_cb)); add_countme_flag(download_data, *download_data.handle); // We don't set the cbdata here yet because the callback_data vector // can grow as repos are added which could change the addresses of each // cbdata. cbd.lr_target = std::unique_ptr(lr_metadatatarget_new2( download_data.handle->get(), NULL, static_cast(progress_cb), static_cast(mirror_failure_cb), NULL, config.get_repo_gpgcheck_option().get_value() ? download_data.pgp.get_keyring_dir().c_str() : NULL, 0)); } catch (const std::runtime_error & e) { libdnf5::throw_with_nested(RepoDownloadError(M_("Failed to set up metadata download"))); } std::tuple>, bool> RepoDownloader::download_repos_descriptions() try { if (callback_data.empty()) { return {}; } GSList * list{nullptr}; for (auto & cbd : callback_data) { // If there is currently no metadata it cannot be up to date if (cbd.repo->get_download_data().get_metadata_path(RepoDownloader::MD_FILENAME_PRIMARY).empty()) { continue; } // Download only metalink or repomd auto & download_data = cbd.repo->get_download_data(); auto & config = cbd.repo->get_config(); if (!config.get_metalink_option().empty() && !config.get_metalink_option().get_value().empty()) { download_data.handle->set_opt(LRO_FETCHMIRRORS, 1L); } else { const char * dlist[] = LR_YUM_REPOMDONLY; download_data.handle->set_opt(LRO_YUMDLIST, dlist); } cbd.lr_target->endcb = end_cb_sync_check; cbd.lr_target->cbdata = &cbd; download_data.handle->set_opt(LRO_FASTESTMIRRORDATA, &cbd); list = g_slist_prepend(list, cbd.lr_target.get()); } std::unique_ptr list_holder(list, &g_slist_free); GError * err{nullptr}; if (!lr_download_metadata(list, &err)) { throw LibrepoError(std::unique_ptr(err)); } std::unordered_map> per_repo_errors; bool all_in_sync = true; for (auto & cbd : callback_data) { std::vector errors; for (GList * err = cbd.lr_target->err; err; err = g_list_next(err)) { gchar * err_msg = (gchar *)err->data; errors.emplace_back(err_msg); } per_repo_errors[cbd.repo.get()] = errors; all_in_sync &= cbd.is_in_sync; } return {per_repo_errors, all_in_sync}; } catch (const std::runtime_error & e) { libdnf5::throw_with_nested(RepoDownloadError(M_("Failed to download metadata"))); } std::unordered_map> RepoDownloader::download() try { if (callback_data.empty()) { return {}; } //TODO(amatej): If we make this API public should we take cacheonly option into account? // PackageDownloader respects it. // For repos it is currently handled in update_and_load_repos so we don't // even attempt to download remote repos with cacheonly enabled. GSList * list{nullptr}; for (auto & cbd : callback_data) { // Don't download repositories that are already in sync if (cbd.is_in_sync) { continue; } auto & download_data = cbd.repo->get_download_data(); // Configure handle to download all the needed metadata download_data.handle->set_opt(LRO_FETCHMIRRORS, 0L); configure_handle_dlist(*download_data.handle, download_data.get_optional_metadata()); cbd.lr_target->endcb = end_cb_full_download; cbd.lr_target->cbdata = &cbd; download_data.handle->set_opt(LRO_FASTESTMIRRORDATA, &cbd); list = g_slist_prepend(list, cbd.lr_target.get()); } std::unique_ptr list_holder(list, &g_slist_free); GError * err{nullptr}; if (!lr_download_metadata(list, &err)) { throw LibrepoError(std::unique_ptr(err)); } std::unordered_map> per_repo_errors; for (auto & cbd : callback_data) { std::vector errors; for (GList * err = cbd.lr_target->err; err; err = g_list_next(err)) { gchar * err_msg = (gchar *)err->data; errors.emplace_back(err_msg); } per_repo_errors[cbd.repo.get()] = errors; } return per_repo_errors; } catch (const std::runtime_error & e) { libdnf5::throw_with_nested(RepoDownloadError(M_("Failed to download metadata"))); } // Use metalink to check whether our metadata are still current. bool RepoDownloader::is_metalink_in_sync(Repo & repo, LrMetalink * metalink) try { auto & download_data = repo.get_download_data(); auto & logger = *(download_data.base)->get_logger(); auto & config = download_data.config; if (!metalink) { logger.trace("Sync check: repo \"{}\" skipped, no metalink", config.get_id()); return false; } // check all recognized hashes auto chksum_free = [](Chksum * ptr) { solv_chksum_free(ptr, nullptr); }; struct hashInfo { const LrMetalinkHash * lr_metalink_hash; std::unique_ptr chksum; }; std::vector hashes; for (auto hash = metalink->hashes; hash; hash = hash->next) { auto lr_metalink_hash = static_cast(hash->data); for (auto algorithm : RECOGNIZED_CHKSUMS) { if (strcmp(lr_metalink_hash->type, algorithm) == 0) hashes.push_back({lr_metalink_hash, {nullptr, chksum_free}}); } } if (hashes.empty()) { logger.trace("Sync check: repo \"{}\" skipped, no usable hash", config.get_id()); return false; } for (auto & hash : hashes) { auto chk_type = solv_chksum_str2type(hash.lr_metalink_hash->type); hash.chksum.reset(solv_chksum_create(chk_type)); } std::ifstream repomd(download_data.repomd_filename, std::ifstream::binary); char buf[4096]; int n_read; while ((n_read = static_cast(repomd.readsome(buf, sizeof(buf)))) > 0) { for (auto & hash : hashes) solv_chksum_add(hash.chksum.get(), buf, n_read); } for (auto & hash : hashes) { int chksumLen; auto chksum = solv_chksum_get(hash.chksum.get(), &chksumLen); std::vector chksumHex(static_cast(chksumLen * 2 + 1), '\0'); solv_bin2hex(chksum, chksumLen, chksumHex.data()); if (strcmp(chksumHex.data(), hash.lr_metalink_hash->value) != 0) { logger.trace( "Sync check: failed for repo \"{}\", {} checksum mismatch", config.get_id(), hash.lr_metalink_hash->type); return false; } } logger.debug("Sync check: repo \"{}\" in sync, metalink checksums match", config.get_id()); return true; } catch (const std::runtime_error & e) { libdnf5::throw_with_nested(RepoDownloadError( M_("Error checking if metalink \"{}\" is in sync for repository \"{}\""), repo.get_download_data().get_source_info().second, repo.get_download_data().config.get_id())); } // Use repomd to check whether our metadata are still current. bool RepoDownloader::is_repomd_in_sync(Repo & repo, std::filesystem::path repomd) try { auto & download_data = repo.get_download_data(); auto & logger = *(download_data.base)->get_logger(); auto & config = download_data.config; auto same = utils::fs::have_files_same_content_noexcept(download_data.repomd_filename.c_str(), repomd.c_str()); if (same) logger.debug("Sync check: repo \"{}\" in sync, repomd matches", config.get_id()); else logger.trace("Sync check: failed for repo \"{}\", repomd mismatch", config.get_id()); return same; } catch (const std::runtime_error & e) { auto src = repo.get_download_data().get_source_info(); libdnf5::throw_with_nested(RepoDownloadError( M_("Error checking if repomd ({}: \"{}\") is in sync for repository \"{}\""), src.first, src.second, repo.get_config().get_id())); } void RepoDownloader::load_local(DownloadData & download_data) try { auto & config = download_data.config; LibrepoHandle h(RepoDownloader::init_local_handle(download_data)); if (config.get_repo_gpgcheck_option().get_value()) { auto pubringdir = download_data.pgp.get_keyring_dir(); h.set_opt(LRO_GNUPGHOMEDIR, pubringdir.c_str()); } auto result = h.perform(); download_data.repomd_filename = libdnf5::utils::string::c_to_str(get_yum_repo(result)->repomd); // copy the mirrors out of the handle (handle_get_info() allocates new // space and passes ownership, so we copy twice in this case, as we want to // store a vector of strings) char ** lr_mirrors; h.get_info(LRI_MIRRORS, &lr_mirrors); if (lr_mirrors) { for (auto mirror = lr_mirrors; *mirror; ++mirror) { download_data.mirrors.emplace_back(*mirror); } } g_strfreev(lr_mirrors); download_data.revision = libdnf5::utils::string::c_to_str(get_yum_repomd(result)->revision); GError * err_p{nullptr}; // TODO(lukash) return time_t instead of converting to signed int download_data.max_timestamp = static_cast(lr_yum_repomd_get_highest_timestamp(get_yum_repomd(result), &err_p)); if (err_p != nullptr) { throw libdnf5::repo::LibrepoError(std::unique_ptr(err_p)); } for (auto elem = get_yum_repomd(result)->content_tags; elem; elem = g_slist_next(elem)) { if (elem->data) { download_data.content_tags.emplace_back(static_cast(elem->data)); } } for (auto elem = get_yum_repomd(result)->distro_tags; elem; elem = g_slist_next(elem)) { if (elem->data) { auto distro_tag = static_cast(elem->data); if (distro_tag->tag) { download_data.distro_tags.emplace_back(distro_tag->cpeid, distro_tag->tag); } } } for (auto elem = get_yum_repomd(result)->records; elem; elem = g_slist_next(elem)) { if (elem->data) { auto rec = static_cast(elem->data); download_data.metadata_locations.emplace_back(rec->type, rec->location_href); } } for (auto * elem = get_yum_repo(result)->paths; elem; elem = g_slist_next(elem)) { if (elem->data) { auto yumrepopath = static_cast(elem->data); download_data.metadata_paths[yumrepopath->type] = yumrepopath->path; } } } catch (const std::runtime_error & e) { libdnf5::throw_with_nested( RepoDownloadError(M_("Error loading local metadata for repository \"{}\""), download_data.config.get_id())); } /// Returns a librepo handle, set as per the repo options. /// Note that destdir is None, and the handle is cached. // TODO(jrohel) The librepo handle callbacks are not set. If librepo itself downloads an extra file // (eg metalink) we won't know about it. LibrepoHandle & RepoDownloader::get_cached_handle(Repo & repo) { auto & donwload_data = repo.get_download_data(); if (!donwload_data.handle) { donwload_data.handle = RepoDownloader::init_remote_handle(repo, nullptr, true); } apply_http_headers(donwload_data, *donwload_data.handle); return *donwload_data.handle; } LibrepoHandle RepoDownloader::init_local_handle(const DownloadData & download_data) { LibrepoHandle h; common_handle_setup(h, download_data); std::string cachedir = download_data.config.get_cachedir(); h.set_opt(LRO_DESTDIR, cachedir.c_str()); const char * urls[] = {cachedir.c_str(), nullptr}; h.set_opt(LRO_URLS, urls); h.set_opt(LRO_LOCAL, 1L); return h; } LibrepoHandle RepoDownloader::init_remote_handle(Repo & repo, const char * destdir, bool mirror_setup) { auto & download_data = repo.get_download_data(); LibrepoHandle h; h.init_remote(download_data.config); RepoDownloader::common_handle_setup(h, download_data); apply_http_headers(download_data, h); h.set_opt(LRO_DESTDIR, destdir); auto & config = download_data.config; enum class Source { NONE, METALINK, MIRRORLIST } source{Source::NONE}; std::string tmp; if (!config.get_metalink_option().empty() && !(tmp = config.get_metalink_option().get_value()).empty()) { source = Source::METALINK; } else if (!config.get_mirrorlist_option().empty() && !(tmp = config.get_mirrorlist_option().get_value()).empty()) { source = Source::MIRRORLIST; } if (source != Source::NONE) { if (mirror_setup) { if (source == Source::METALINK) { h.set_opt(LRO_METALINKURL, tmp.c_str()); } else { h.set_opt(LRO_MIRRORLISTURL, tmp.c_str()); // YUM-DNF compatibility hack. YUM guessed by content of keyword "metalink" if // mirrorlist is really mirrorlist or metalink) if (tmp.find("metalink") != tmp.npos) h.set_opt(LRO_METALINKURL, tmp.c_str()); } h.set_opt(LRO_FASTESTMIRROR, config.get_fastestmirror_option().get_value() ? 1L : 0L); auto fastest_mirror_cache_dir = config.get_basecachedir_option().get_value(); if (fastest_mirror_cache_dir.back() != '/') { fastest_mirror_cache_dir.push_back('/'); } fastest_mirror_cache_dir += "fastestmirror.cache"; h.set_opt(LRO_FASTESTMIRRORCACHE, fastest_mirror_cache_dir.c_str()); } else { // use already resolved mirror list std::vector c_mirrors(download_data.mirrors.size() + 1, nullptr); str_vector_to_char_array(download_data.mirrors, c_mirrors.data()); h.set_opt(LRO_URLS, c_mirrors.data()); } } else if (!config.get_baseurl_option().get_value().empty()) { std::vector urls(config.get_baseurl_option().get_value().size() + 1, nullptr); str_vector_to_char_array(config.get_baseurl_option().get_value(), urls.data()); h.set_opt(LRO_URLS, urls.data()); } else { throw RepoDownloadError( M_("No valid source (baseurl, mirrorlist or metalink) found for repository \"{}\""), config.get_id()); } return h; } void RepoDownloader::configure_handle_dlist(LibrepoHandle & h, std::set && optional_metadata) { if (optional_metadata.extract(libdnf5::METADATA_TYPE_ALL)) { h.set_opt(LRO_YUMDLIST, LR_RPMMD_FULL); } else { std::vector dlist; dlist.push_back(MD_FILENAME_PRIMARY); #ifdef WITH_MODULEMD dlist.push_back(MD_FILENAME_MODULES); #endif if (optional_metadata.extract(libdnf5::METADATA_TYPE_FILELISTS)) { dlist.push_back(MD_FILENAME_FILELISTS); } if (optional_metadata.extract(libdnf5::METADATA_TYPE_OTHER)) { dlist.push_back(MD_FILENAME_OTHER); } if (optional_metadata.extract(libdnf5::METADATA_TYPE_PRESTO)) { dlist.push_back(MD_FILENAME_PRESTODELTA); } if (optional_metadata.extract(libdnf5::METADATA_TYPE_COMPS)) { dlist.push_back(MD_FILENAME_GROUP_GZ); } if (optional_metadata.extract(libdnf5::METADATA_TYPE_UPDATEINFO)) { dlist.push_back(MD_FILENAME_UPDATEINFO); } // download the rest metadata added by 3rd parties for (auto & item : optional_metadata) { // the appstream metadata is a "virtual" type, the list // of the types is read from the repomd file if (item.compare(libdnf5::METADATA_TYPE_APPSTREAM) != 0) dlist.push_back(item.c_str()); } if (optional_metadata.extract(libdnf5::METADATA_TYPE_APPSTREAM)) { // ideally, the repomd.xml file should be read and every type matching is_appstream_metadata_type() // would be added from it, but the content is not known at this point and when it is known, then // it's too late, thus declare some "expected" types to be downloaded here dlist.push_back("appstream"); dlist.push_back("appstream-icons"); dlist.push_back("appstream-icons-48x48"); dlist.push_back("appstream-icons-48x48@2"); dlist.push_back("appstream-icons-64x64"); dlist.push_back("appstream-icons-64x64@2"); dlist.push_back("appstream-icons-128x128"); dlist.push_back("appstream-icons-128x128@2"); // consult the prefixes with the is_appstream_metadata_type() dlist.push_back("appdata"); dlist.push_back("appdata-icons"); dlist.push_back("appdata-icons-48x48"); dlist.push_back("appdata-icons-48x48@2"); dlist.push_back("appdata-icons-64x64"); dlist.push_back("appdata-icons-64x64@2"); dlist.push_back("appdata-icons-128x128"); dlist.push_back("appdata-icons-128x128@2"); } dlist.push_back(nullptr); h.set_opt(LRO_YUMDLIST, dlist.data()); } } void RepoDownloader::common_handle_setup(LibrepoHandle & h, const DownloadData & download_data) { configure_handle_dlist(h, download_data.get_optional_metadata()); h.set_opt(LRO_PRESERVETIME, static_cast(download_data.preserve_remote_time)); h.set_opt(LRO_REPOTYPE, LR_YUMREPO); h.set_opt(LRO_INTERRUPTIBLE, 1L); h.set_opt(LRO_GPGCHECK, download_data.config.get_repo_gpgcheck_option().get_value()); h.set_opt(LRO_MAXMIRRORTRIES, static_cast(download_data.max_mirror_tries)); h.set_opt(LRO_MAXPARALLELDOWNLOADS, download_data.config.get_max_parallel_downloads_option().get_value()); h.set_opt(LRO_MAXDOWNLOADSPERMIRROR, download_data.config.get_max_downloads_per_mirror_option().get_value()); LrUrlVars * repomd_substs = nullptr; repomd_substs = lr_urlvars_set(repomd_substs, MD_FILENAME_GROUP_GZ, MD_FILENAME_GROUP); h.set_opt(LRO_YUMSLIST, repomd_substs); LrUrlVars * substs = nullptr; // Deprecated, we have direct access to base, therefore we don't need to set explicitly substitutions for (const auto & item : download_data.substitutions) { substs = lr_urlvars_set(substs, item.first.c_str(), item.second.c_str()); } for (const auto & item : download_data.base->get_vars()->get_variables()) { substs = lr_urlvars_set(substs, item.first.c_str(), item.second.value.c_str()); } h.set_opt(LRO_VARSUB, substs); #ifdef LRO_SUPPORTS_CACHEDIR // If zchunk is enabled, set librepo cache dir if (download_data.config.get_main_config().get_zchunk_option().get_value()) { h.set_opt(LRO_CACHEDIR, download_data.config.get_basecachedir_option().get_value().c_str()); } #endif } void RepoDownloader::apply_http_headers(DownloadData & download_data, LibrepoHandle & handle) { if (download_data.http_headers.empty()) { handle.set_opt(LRO_HTTPHEADER, nullptr); return; } std::unique_ptr> lr_headers { nullptr, [](char ** ptr) { for (auto item = ptr; *item != nullptr; ++item) { delete[] *item; } delete[] ptr; } }; lr_headers.reset(new char * [download_data.http_headers.size() + 1] {}); for (size_t i = 0; i < download_data.http_headers.size(); ++i) { const auto & header = download_data.http_headers[i]; lr_headers[i] = new char[header.size() + 1]; strcpy(lr_headers[i], header.c_str()); } handle.set_opt(LRO_HTTPHEADER, lr_headers.get()); } // COUNTME CONSTANTS // // width of the sliding time window (in seconds) const int COUNTME_WINDOW = 7 * 24 * 60 * 60; // 1 week // starting point of the sliding time window relative to the UNIX epoch // allows for aligning the window with a specific weekday const int COUNTME_OFFSET = 345600; // Monday (1970-01-05 00:00:00 UTC) // estimated number of metalink requests sent over the window // used to generate the probability distribution of counting events const int COUNTME_BUDGET = 4; // metadata_expire defaults to 2 days // cookie file name const std::string COUNTME_COOKIE = "countme"; // cookie file format version const int COUNTME_VERSION = 0; // longevity buckets that we report in the flag // example: {A, B, C} defines 4 buckets [0, A), [A, B), [B, C), [C, infinity) // where each letter represents a window step (starting from 0) const std::array COUNTME_BUCKETS = {{2, 5, 25}}; /// The countme flag will be added once (and only once) in every position of /// a sliding time window (COUNTME_WINDOW) that starts at COUNTME_OFFSET and /// moves along the time axis, by one length at a time, in such a way that /// the current point in time always stays within: /// /// UNIX epoch now /// | | /// |---*-----|-----|-----|-----[-*---]---> time /// | ~~~~~~~ /// COUNTME_OFFSET COUNTME_WINDOW /// /// This is to align the time window with an absolute point in time rather /// than the last counting event (which could facilitate tracking across /// multiple such events). /// /// In the below comments, the window's current position will be referred to /// as "this window" for brevity. void RepoDownloader::add_countme_flag(DownloadData & download_data, LibrepoHandle & handle) { auto & logger = *(download_data.base)->get_logger(); auto & config = download_data.config; // Bail out if not counting or not running as root (since the persistdir is // only root-writable) if (!config.get_countme_option().get_value() || getuid() != 0) return; // Bail out if not a remote handle long local; handle.get_info(LRI_LOCAL, &local); if (local) return; // Bail out if no metalink or mirrorlist is defined auto & metalink = config.get_metalink_option(); auto & mirrorlist = config.get_mirrorlist_option(); if ((metalink.empty() || metalink.get_value().empty()) && (mirrorlist.empty() || mirrorlist.get_value().empty())) return; if (!std::filesystem::is_directory(config.get_persistdir())) { std::filesystem::create_directories(config.get_persistdir()); } // Load the cookie std::filesystem::path file_path(config.get_persistdir()); file_path /= COUNTME_COOKIE; int ver = COUNTME_VERSION; // file format version (for future use) time_t epoch = 0; // position of first observed window time_t win = COUNTME_OFFSET; // position of last counted window int budget = -1; // budget for this window (-1 = generate) // TODO(lukash) ideally replace with utils::fs::File (via adding scanf() support?), // once we are able to test this (using CI stack tests) std::ifstream(file_path) >> ver >> epoch >> win >> budget; // Bail out if the window has not advanced since time_t now = time(nullptr); time_t delta = now - win; if (delta < COUNTME_WINDOW) { logger.trace("countme: no event for repo \"{}\": window already counted", config.get_id()); return; } // Evenly distribute the probability of the counting event over the first N // requests in this window (where N = COUNTME_BUDGET), by defining a random // "budget" of ordinary requests that we first have to spend. This ensures // that no particular request is special and thus no privacy loss is // incurred by adding the flag within N requests. if (budget < 0) { std::random_device rd; std::default_random_engine gen(rd()); std::uniform_int_distribution dist(1, COUNTME_BUDGET); budget = dist(gen); } budget--; if (!budget) { // Budget exhausted, counting! // Compute the position of this window win = now - (delta % COUNTME_WINDOW); // Compute the epoch from this system's epoch or, if unknown, declare // this window as the epoch (unless stored in the cookie previously). time_t sysepoch = get_system_epoch(); if (sysepoch) epoch = sysepoch - ((sysepoch - COUNTME_OFFSET) % COUNTME_WINDOW); if (!epoch) epoch = win; // Window step (0 at epoch) int64_t step = (win - epoch) / COUNTME_WINDOW; // Compute the bucket we are in uint32_t i; for (i = 0; i < COUNTME_BUCKETS.size(); ++i) if (step < COUNTME_BUCKETS[i]) break; uint32_t bucket = i + 1; // Buckets are numbered from 1 // Set the flag std::string flag = "countme=" + std::to_string(bucket); handle.set_opt(LRO_ONETIMEFLAG, flag.c_str()); logger.trace("countme: event triggered for repo \"{}\": bucket {}", config.get_id(), bucket); // Request a new budget budget = -1; } else { logger.trace("countme: no event for repo \"{}\": budget to spend: {}", config.get_id(), budget); } // Save the cookie utils::fs::File(file_path, "w").write(fmt::format("{} {} {} {}", COUNTME_VERSION, epoch, win, budget)); } /* Returns this system's installation time ("epoch") as a UNIX timestamp. * * Uses the machine-id(5) file's mtime as a good-enough source of truth. This * file is typically tied to the system's installation or first boot where it's * populated by an installer tool or init system, respectively, and is never * changed afterwards. * * Some systems, such as containers that don't run an init system, may have the * file missing, empty or uninitialized, in which case this function returns 0. */ time_t RepoDownloader::get_system_epoch() { std::string filename = "/etc/machine-id"; std::string id; struct stat st; if (stat(filename.c_str(), &st) != 0 || !st.st_size) return 0; std::ifstream(filename) >> id; if (id == "uninitialized") return 0; return st.st_mtime; } } // namespace libdnf5::repo