folkslinux
/
mamba
mirror of https://github.com/mamba-org/mamba.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor `SubdirData` > `SubdirIndexLoader` (#3940)

This commit is contained in:
Antoine Prouvost 2025-05-15 16:57:52 +02:00 committed by GitHub
parent e3ab63ac45
commit 06000139c9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 773 additions and 273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
libmamba/CMakeLists.txt
View File

@ -244,7 +244,7 @@ set(
${LIBMAMBA_SOURCE_DIR}/core/run.cpp
${LIBMAMBA_SOURCE_DIR}/core/shell_init.cpp
${LIBMAMBA_SOURCE_DIR}/core/singletons.cpp
${LIBMAMBA_SOURCE_DIR}/core/subdirdata.cpp
${LIBMAMBA_SOURCE_DIR}/core/subdir_index.cpp
${LIBMAMBA_SOURCE_DIR}/core/thread_utils.cpp
${LIBMAMBA_SOURCE_DIR}/core/timeref.cpp
${LIBMAMBA_SOURCE_DIR}/core/transaction_context.cpp
@ -391,7 +391,7 @@ set(
${LIBMAMBA_INCLUDE_DIR}/mamba/core/repo_checker_store.hpp
${LIBMAMBA_INCLUDE_DIR}/mamba/core/run.hpp
${LIBMAMBA_INCLUDE_DIR}/mamba/core/shell_init.hpp
${LIBMAMBA_INCLUDE_DIR}/mamba/core/subdirdata.hpp
${LIBMAMBA_INCLUDE_DIR}/mamba/core/subdir_index.hpp
${LIBMAMBA_INCLUDE_DIR}/mamba/core/tasksync.hpp
${LIBMAMBA_INCLUDE_DIR}/mamba/core/thread_utils.hpp
${LIBMAMBA_INCLUDE_DIR}/mamba/core/timeref.hpp

28
libmamba/include/mamba/core/context.hpp
View File

@ -208,11 +208,33 @@ namespace mamba
SubdirParams subdir_params() const
{
const auto get_local_repodata_ttl = [&]() -> std::optional<std::size_t>
{
// Force the use of index cache by setting TTL to 0
if (this->use_index_cache)
{
return { 0 };
}
// This is legacy where from where 1 meant to read from header
if (this->local_repodata_ttl == 1)
{
return std::nullopt;
}
return { this->local_repodata_ttl };
};
return {
/* .local_repodata_ttl */ this->local_repodata_ttl,
/* .local_repodata_ttl */ get_local_repodata_ttl(),
/* .offline */ this->offline,
/* .use_index_cache */ this->use_index_cache,
/* .repodata_use_zst */ this->repodata_use_zst,
/* .force_use_zst */ false // Must override based on ChannelContext
};
}
SubdirDownloadParams subdir_download_params() const
{
return {
/* .offline */ this->offline,
/* .repodata_check_zst */ this->repodata_use_zst,
};
}

14
libmamba/include/mamba/core/download_progress_bar.hpp
View File

@ -23,20 +23,20 @@ namespace mamba
bool no_clear_progress_bar = false;
};
class SubdirDataMonitor : public download::Monitor
class SubdirIndexMonitor : public download::Monitor
{
public:
static bool can_monitor(const Context& context);
explicit SubdirDataMonitor(MonitorOptions options = {});
virtual ~SubdirDataMonitor() = default;
explicit SubdirIndexMonitor(MonitorOptions options = {});
virtual ~SubdirIndexMonitor() = default;
SubdirDataMonitor(const SubdirDataMonitor&) = delete;
SubdirDataMonitor& operator=(const SubdirDataMonitor&) = delete;
SubdirIndexMonitor(const SubdirIndexMonitor&) = delete;
SubdirIndexMonitor& operator=(const SubdirIndexMonitor&) = delete;
SubdirDataMonitor(SubdirDataMonitor&&) = delete;
SubdirDataMonitor& operator=(SubdirDataMonitor&&) = delete;
SubdirIndexMonitor(SubdirIndexMonitor&&) = delete;
SubdirIndexMonitor& operator=(SubdirIndexMonitor&&) = delete;
void reset_options(MonitorOptions options);

4
libmamba/include/mamba/core/package_database_loader.hpp
View File

@ -15,7 +15,7 @@ namespace mamba
{
class Context;
class PrefixData;
class SubdirData;
class SubdirIndexLoader;
namespace solver::libsolv
{
@ -27,7 +27,7 @@ namespace mamba
auto load_subdir_in_database( //
const Context& ctx,
solver::libsolv::Database& database,
const SubdirData& subdir
const SubdirIndexLoader& subdir
) -> expected_t<solver::libsolv::RepoInfo>;
auto load_installed_packages_in_database(

126
libmamba/include/mamba/core/subdirdata.hpp → libmamba/include/mamba/core/subdir_index.hpp
View File

@ -10,6 +10,7 @@
#include <algorithm>
#include <optional>
#include <string>
#include <type_traits>
#include <nlohmann/json_fwd.hpp>
@ -30,8 +31,6 @@ namespace mamba
class Channel;
}
class ChannelContext;
/**
* Handling of a subdirectory metadata.
*
@ -117,10 +116,10 @@ namespace mamba
*
* Upon creation, the caches are checked for a valid and up to date index.
* This can be inspected with @ref valid_cache_found.
* The created subdirs are typically used with @ref SubdirData::download_required_indexes
* The created subdirs are typically used with @ref SubdirIndexLoader::download_required_indexes
* which will download the missing, invalid, or outdated indexes as needed.
*/
class SubdirData
class SubdirIndexLoader
{
public:
@ -135,7 +134,7 @@ namespace mamba
[[nodiscard]] static auto download_required_indexes(
SubdirIter1 subdirs_first,
SubdirIter2 subdirs_last,
const SubdirParams& subdir_params,
const SubdirDownloadParams& subdir_params,
const specs::AuthenticationDataBase& auth_info,
const download::mirror_map& mirrors,
const download::Options& download_options,
@ -146,7 +145,7 @@ namespace mamba
template <typename Subdirs>
[[nodiscard]] static auto download_required_indexes(
Subdirs& subdirs,
const SubdirParams& subdir_params,
const SubdirDownloadParams& subdir_params,
const specs::AuthenticationDataBase& auth_info,
const download::mirror_map& mirrors,
const download::Options& download_options,
@ -158,12 +157,11 @@ namespace mamba
/** Check existing caches for a valid index validity and freshness. */
static auto create(
const SubdirParams& params,
ChannelContext& channel_context,
specs::Channel channel,
specs::DynamicPlatform platform,
MultiPackageCache& caches,
std::string repodata_filename = "repodata.json"
) -> expected_t<SubdirData>;
) -> expected_t<SubdirIndexLoader>;
[[nodiscard]] auto is_noarch() const -> bool;
[[nodiscard]] auto is_local() const -> bool;
@ -180,14 +178,21 @@ namespace mamba
[[nodiscard]] auto writable_libsolv_cache_path() const -> fs::u8path;
[[nodiscard]] auto valid_json_cache_path() const -> expected_t<fs::u8path>;
void clear_cache_files();
void clear_valid_cache_files();
private:
// This paths are pointing to what is found when iterating over the cache directories.
// The expired found is the first one, which could be improved by keeping the freshest one.
// This could improve caching in some HTTP 304 cases.
// A possible improvement would be to keep all path, metadatas, and writable status in a
// single vector and sort them by recency.
// This would also give a public option for `clear`-ing all writable caches, not just the
// valid one.
SubdirMetadata m_metadata;
specs::Channel m_channel;
fs::u8path m_valid_cache_path;
fs::u8path m_expired_cache_path;
std::optional<fs::u8path> m_expired_cache_path;
fs::u8path m_writable_pkgs_dir;
specs::DynamicPlatform m_platform;
std::string m_repodata_filename;
@ -197,9 +202,8 @@ namespace mamba
bool m_json_cache_valid = false;
bool m_solv_cache_valid = false;
SubdirData(
SubdirIndexLoader(
const SubdirParams& params,
ChannelContext& channel_context,
specs::Channel channel,
std::string platform,
MultiPackageCache& caches,
@ -207,27 +211,20 @@ namespace mamba
);
[[nodiscard]] auto repodata_url_path() const -> std::string;
[[nodiscard]] auto valid_json_cache_path_unchecked() const -> fs::u8path;
[[nodiscard]] auto valid_state_file_path_unchecked() const -> fs::u8path;
[[nodiscard]] auto valid_libsolv_cache_path_unchecked() const -> fs::u8path;
/**************************************************
* Implementation details of SubdirData::create *
**************************************************/
/*********************************************************
* Implementation details of SubdirIndexLoader::create *
*********************************************************/
void load(
const MultiPackageCache& caches,
ChannelContext& channel_context,
const SubdirParams& params,
const specs::Channel& channel
);
void load(const MultiPackageCache& caches, const SubdirParams& params);
void load_cache(const MultiPackageCache& caches, const SubdirParams& params);
void update_metadata_zst(
ChannelContext& context,
const SubdirParams& params,
const specs::Channel& channel
);
/*********************************************************************
* Implementation details of SubdirData::download_required_indexes *
*********************************************************************/
/****************************************************************************
* Implementation details of SubdirIndexLoader::download_required_indexes *
****************************************************************************/
auto use_existing_cache() -> expected_t<void>;
auto finalize_transfer(SubdirMetadata::HttpMetadata http_data, const fs::u8path& artifact)
@ -236,15 +233,16 @@ namespace mamba
template <typename First, typename End>
static auto
build_all_check_requests(First subdirs_first, End subdirs_last, const SubdirParams& params)
build_all_check_requests(First subdirs_first, End subdirs_last, const SubdirDownloadParams& params)
-> download::MultiRequest;
auto build_check_requests(const SubdirParams& params) -> download::MultiRequest;
auto build_check_requests(const SubdirDownloadParams& params) -> download::MultiRequest;
template <typename First, typename End>
static auto
build_all_index_requests(First subdirs_first, End subdirs_last, const SubdirParams& params)
build_all_index_requests(First subdirs_first, End subdirs_last, const SubdirDownloadParams& params)
-> download::MultiRequest;
auto build_index_request(const SubdirParams& params) -> std::optional<download::Request>;
auto build_index_request(const SubdirDownloadParams& params)
-> std::optional<download::Request>;
[[nodiscard]] static auto download_requests(
download::MultiRequest index_requests,
@ -282,15 +280,15 @@ namespace mamba
*/
auto create_cache_dir(const fs::u8path& cache_path) -> std::string;
/**********************************
* Implementation of Subdirdata *
**********************************/
/*****************************************
* Implementation of SubdirIndexLoader *
*****************************************/
template <typename SubdirIter1, typename SubdirIter2>
auto SubdirData::download_required_indexes(
auto SubdirIndexLoader::download_required_indexes(
SubdirIter1 subdirs_first,
SubdirIter2 subdirs_last,
const SubdirParams& subdir_params,
const SubdirDownloadParams& subdir_params,
const specs::AuthenticationDataBase& auth_info,
const download::mirror_map& mirrors,
const download::Options& download_options,
@ -328,9 +326,9 @@ namespace mamba
}
template <typename Subdirs>
auto SubdirData::download_required_indexes(
auto SubdirIndexLoader::download_required_indexes(
Subdirs& subdirs,
const SubdirParams& subdir_params,
const SubdirDownloadParams& subdir_params,
const specs::AuthenticationDataBase& auth_info,
const download::mirror_map& mirrors,
const download::Options& download_options,
@ -353,16 +351,30 @@ namespace mamba
}
template <typename First, typename End>
auto
SubdirData::build_all_check_requests(First subdirs_first, End subdirs_last, const SubdirParams& params)
-> download::MultiRequest
auto SubdirIndexLoader::build_all_check_requests(
First subdirs_first,
End subdirs_last,
const SubdirDownloadParams& params
) -> download::MultiRequest
{
download::MultiRequest requests;
for (; subdirs_first != subdirs_last; ++subdirs_first)
{
if (!subdirs_first->valid_cache_found())
// TODO(C++23): We make a special handling of iterators of pointers due to the
// difficulty and necessity to create a range of references from Python objects.
SubdirIndexLoader* p_subdir = nullptr;
if constexpr (std::is_pointer_v<std::remove_reference_t<decltype(*subdirs_first)>>)
{
auto check_list = subdirs_first->build_check_requests(params);
p_subdir = *subdirs_first;
}
else
{
p_subdir = &(*subdirs_first);
}
if (p_subdir != nullptr && !p_subdir->valid_cache_found())
{
auto check_list = p_subdir->build_check_requests(params);
std::move(check_list.begin(), check_list.end(), std::back_inserter(requests));
}
}
@ -370,16 +382,30 @@ namespace mamba
}
template <typename First, typename End>
auto
SubdirData::build_all_index_requests(First subdirs_first, End subdirs_last, const SubdirParams& params)
-> download::MultiRequest
auto SubdirIndexLoader::build_all_index_requests(
First subdirs_first,
End subdirs_last,
const SubdirDownloadParams& params
) -> download::MultiRequest
{
download::MultiRequest requests;
for (; subdirs_first != subdirs_last; ++subdirs_first)
{
if (!subdirs_first->valid_cache_found())
// TODO(C++23): We make a special handling of iterators of pointers due to the
// difficulty and necessity to create a range of references from Python objects.
SubdirIndexLoader* p_subdir = nullptr;
if constexpr (std::is_pointer_v<std::remove_reference_t<decltype(*subdirs_first)>>)
{
if (auto request = subdirs_first->build_index_request(params))
p_subdir = *subdirs_first;
}
else
{
p_subdir = &(*subdirs_first);
}
if (!p_subdir->valid_cache_found())
{
if (auto request = p_subdir->build_index_request(params))
{
requests.push_back(*std::move(request));
}

20
libmamba/include/mamba/core/subdir_parameters.hpp
View File

@ -7,16 +7,28 @@
#ifndef MAMBA_CORE_SUBDIR_PARAMETERS_HPP
#define MAMBA_CORE_SUBDIR_PARAMETERS_HPP
#include <cstddef>
#include <optional>
namespace mamba
{
struct SubdirParams
{
std::size_t local_repodata_ttl = 1;
/**
* Repodata cache time to live in seconds.
*
* If not specified, then it is read from server headers.
*/
std::optional<std::size_t> local_repodata_ttl_s = std::nullopt;
bool offline = false;
bool use_index_cache = true;
bool repodata_use_zst = true;
/** Force the use of zst for this subdir without checking. */
bool repodata_force_use_zst = false;
};
struct SubdirDownloadParams
{
bool offline = false;
/** Make a request to check the use of zst compression format. */
bool repodata_check_zst = true;
};
}

34
libmamba/src/api/channel_loader.cpp
View File

@ -10,7 +10,7 @@
#include "mamba/core/output.hpp"
#include "mamba/core/package_database_loader.hpp"
#include "mamba/core/prefix_data.hpp"
#include "mamba/core/subdirdata.hpp"
#include "mamba/core/subdir_index.hpp"
#include "mamba/solver/libsolv/database.hpp"
#include "mamba/solver/libsolv/repo_info.hpp"
#include "mamba/specs/package_info.hpp"
@ -54,7 +54,7 @@ namespace mamba
ChannelContext& channel_context,
const specs::Channel& channel,
MultiPackageCache& package_caches,
std::vector<SubdirData>& subdirs,
std::vector<SubdirIndexLoader>& subdirs,
std::vector<mamba_error>& error_list,
std::vector<solver::libsolv::Priorities>& priorities,
int& max_prio,
@ -76,9 +76,10 @@ namespace mamba
LOG_WARNING << "See: https://legal.anaconda.com/policies/en/";
}
auto sdires = SubdirData::create(
ctx.subdir_params(),
channel_context,
auto subdir_params = ctx.subdir_params();
subdir_params.repodata_force_use_zst = channel_context.has_zst(channel);
auto sdires = SubdirIndexLoader::create(
subdir_params,
channel,
platform,
package_caches,
@ -90,6 +91,11 @@ namespace mamba
continue;
}
auto sdir = std::move(sdires).value();
if (sdir.valid_cache_found())
{
Console::stream() << fmt::format("{:<50} {:>20}", sdir.name(), "Using cache");
}
subdirs.push_back(std::move(sdir));
if (ctx.channel_priority == ChannelPriority::Disabled)
{
@ -130,7 +136,7 @@ namespace mamba
bool is_retry
) -> expected_t<void, mamba_aggregated_error>
{
std::vector<SubdirData> subdirs;
std::vector<SubdirIndexLoader> subdirs;
std::vector<solver::libsolv::Priorities> priorities;
int max_prio = static_cast<int>(ctx.channels.size());
@ -200,13 +206,13 @@ namespace mamba
}
expected_t<void> download_res;
if (SubdirDataMonitor::can_monitor(ctx))
if (SubdirIndexMonitor::can_monitor(ctx))
{
SubdirDataMonitor check_monitor({ true, true });
SubdirDataMonitor index_monitor;
download_res = SubdirData::download_required_indexes(
SubdirIndexMonitor check_monitor({ true, true });
SubdirIndexMonitor index_monitor;
download_res = SubdirIndexLoader::download_required_indexes(
subdirs,
ctx.subdir_params(),
ctx.subdir_download_params(),
ctx.authentication_info(),
ctx.mirrors,
ctx.download_options(),
@ -217,9 +223,9 @@ namespace mamba
}
else
{
download_res = SubdirData::download_required_indexes(
download_res = SubdirIndexLoader::download_required_indexes(
subdirs,
ctx.subdir_params(),
ctx.subdir_download_params(),
ctx.authentication_info(),
ctx.mirrors,
ctx.download_options(),
@ -283,7 +289,7 @@ namespace mamba
{
LOG_WARNING << "Could not load repodata.json for " << subdir.name()
<< ". Deleting cache, and retrying.";
subdir.clear_cache_files();
subdir.clear_valid_cache_files();
loading_failed = true;
}
}

32
libmamba/src/core/download_progress_bar.cpp
View File

@ -128,21 +128,21 @@ namespace mamba
}
}
/*********************
* SubdirDataMonitor *
*********************/
/**********************
* SubdirIndexMonitor *
**********************/
SubdirDataMonitor::SubdirDataMonitor(MonitorOptions options)
SubdirIndexMonitor::SubdirIndexMonitor(MonitorOptions options)
: m_options(std::move(options))
{
}
void SubdirDataMonitor::reset_options(MonitorOptions options)
void SubdirIndexMonitor::reset_options(MonitorOptions options)
{
m_options = std::move(options);
}
bool SubdirDataMonitor::can_monitor(const Context& context)
bool SubdirIndexMonitor::can_monitor(const Context& context)
{
return !(
context.graphics_params.no_progress_bars || context.output_params.json
@ -150,7 +150,8 @@ namespace mamba
);
}
void SubdirDataMonitor::observe_impl(download::MultiRequest& requests, download::Options& options)
void
SubdirIndexMonitor::observe_impl(download::MultiRequest& requests, download::Options& options)
{
m_throttle_time.resize(requests.size(), std::chrono::steady_clock::now());
m_progress_bar.reserve(requests.size());
@ -175,7 +176,7 @@ namespace mamba
options.on_unexpected_termination = [this]() { on_unexpected_termination(); };
}
void SubdirDataMonitor::on_done_impl()
void SubdirIndexMonitor::on_done_impl()
{
auto& pbar_manager = Console::instance().progress_bar_manager();
if (pbar_manager.started())
@ -191,22 +192,23 @@ namespace mamba
m_options = MonitorOptions{};
}
void SubdirDataMonitor::on_unexpected_termination_impl()
void SubdirIndexMonitor::on_unexpected_termination_impl()
{
Console::instance().progress_bar_manager().terminate();
}
void SubdirDataMonitor::update_progress_bar(std::size_t index, const download::Event& event)
void SubdirIndexMonitor::update_progress_bar(std::size_t index, const download::Event& event)
{
std::visit([this, index](auto&& arg) { update_progress_bar(index, arg); }, event);
}
void SubdirDataMonitor::update_progress_bar(std::size_t index, const download::Progress& progress)
void
SubdirIndexMonitor::update_progress_bar(std::size_t index, const download::Progress& progress)
{
mamba::update_progress_bar(m_progress_bar[index], m_throttle_time[index], progress);
}
void SubdirDataMonitor::update_progress_bar(std::size_t index, const download::Error& error)
void SubdirIndexMonitor::update_progress_bar(std::size_t index, const download::Error& error)
{
if (m_options.checking_download)
{
@ -218,7 +220,7 @@ namespace mamba
}
}
void SubdirDataMonitor::update_progress_bar(std::size_t index, const download::Success& success)
void SubdirIndexMonitor::update_progress_bar(std::size_t index, const download::Success& success)
{
if (m_options.checking_download)
{
@ -230,7 +232,7 @@ namespace mamba
}
}
void SubdirDataMonitor::complete_checking_progress_bar(std::size_t index)
void SubdirIndexMonitor::complete_checking_progress_bar(std::size_t index)
{
ProgressProxy& progress_bar = m_progress_bar[index];
progress_bar.repr().postfix.set_value("Checked");
@ -245,7 +247,7 @@ namespace mamba
bool PackageDownloadMonitor::can_monitor(const Context& context)
{
return SubdirDataMonitor::can_monitor(context);
return SubdirIndexMonitor::can_monitor(context);
}
PackageDownloadMonitor::~PackageDownloadMonitor()

10
libmamba/src/core/package_database_loader.cpp
View File

@ -17,7 +17,7 @@
#include "mamba/core/output.hpp"
#include "mamba/core/package_database_loader.hpp"
#include "mamba/core/prefix_data.hpp"
#include "mamba/core/subdirdata.hpp"
#include "mamba/core/subdir_index.hpp"
#include "mamba/core/virtual_packages.hpp"
#include "mamba/solver/libsolv/database.hpp"
#include "mamba/solver/libsolv/repo_info.hpp"
@ -52,9 +52,11 @@ namespace mamba
);
}
auto
load_subdir_in_database(const Context& ctx, solver::libsolv::Database& database, const SubdirData& subdir)
-> expected_t<solver::libsolv::RepoInfo>
auto load_subdir_in_database(
const Context& ctx,
solver::libsolv::Database& database,
const SubdirIndexLoader& subdir
) -> expected_t<solver::libsolv::RepoInfo>
{
const auto expected_cache_origin = solver::libsolv::RepodataOrigin{
/* .url= */ util::rsplit(subdir.metadata().url(), "/", 1).front(),

2
libmamba/src/core/repo_checker_store.cpp
View File

@ -9,7 +9,7 @@
#include "mamba/core/output.hpp"
#include "mamba/core/package_cache.hpp"
#include "mamba/core/repo_checker_store.hpp"
#include "mamba/core/subdirdata.hpp"
#include "mamba/core/subdir_index.hpp"
namespace mamba
{

268
libmamba/src/core/subdirdata.cpp → libmamba/src/core/subdir_index.cpp
View File

@ -4,6 +4,7 @@
//
// The full license is in the file LICENSE, distributed with this software.
#include <charconv>
#include <memory>
#include <regex>
#include <stdexcept>
@ -11,7 +12,7 @@
#include "mamba/core/channel_context.hpp"
#include "mamba/core/output.hpp"
#include "mamba/core/package_cache.hpp"
#include "mamba/core/subdirdata.hpp"
#include "mamba/core/subdir_index.hpp"
#include "mamba/core/thread_utils.hpp"
#include "mamba/core/util.hpp"
#include "mamba/fs/filesystem.hpp"
@ -30,7 +31,8 @@ namespace mamba
namespace
{
#ifdef _WIN32
std::chrono::system_clock::time_point filetime_to_unix(const fs::file_time_type& filetime)
auto filetime_to_unix(const fs::file_time_type& filetime)
-> std::chrono::system_clock::time_point
{
// windows filetime is in 100ns intervals since 1601-01-01
static constexpr auto epoch_offset = std::chrono::seconds(11644473600ULL);
@ -47,7 +49,7 @@ namespace mamba
// "_etag": "W/\"6092e6a2b6cec6ea5aade4e177c3edda-8\"",
// "_mod": "Sat, 04 Apr 2020 03:29:49 GMT",
// "_cache_control": "public, max-age=1200"
std::string extract_subjson(std::ifstream& s)
auto extract_subjson(std::ifstream& s) -> std::string
{
char next = {};
std::string result = {};
@ -180,7 +182,7 @@ namespace mamba
out << j.dump(4);
}
bool SubdirMetadata::is_valid_metadata(const fs::u8path& file) const
auto SubdirMetadata::is_valid_metadata(const fs::u8path& file) const -> bool
{
if (const auto new_size = fs::file_size(file); new_size != m_stored_file_size)
{
@ -200,27 +202,27 @@ namespace mamba
return last_write_time_valid;
}
const std::string& SubdirMetadata::url() const
auto SubdirMetadata::url() const -> const std::string&
{
return m_http.url;
}
const std::string& SubdirMetadata::etag() const
auto SubdirMetadata::etag() const -> const std::string&
{
return m_http.etag;
}
const std::string& SubdirMetadata::last_modified() const
auto SubdirMetadata::last_modified() const -> const std::string&
{
return m_http.last_modified;
}
const std::string& SubdirMetadata::cache_control() const
auto SubdirMetadata::cache_control() const -> const std::string&
{
return m_http.cache_control;
}
bool SubdirMetadata::has_up_to_date_zst() const
auto SubdirMetadata::has_up_to_date_zst() const -> bool
{
return m_has_zst.has_value() && m_has_zst.value().value && !m_has_zst.value().has_expired();
}
@ -313,16 +315,16 @@ namespace mamba
}
}
bool SubdirMetadata::CheckedAt::has_expired() const
auto SubdirMetadata::CheckedAt::has_expired() const -> bool
{
// difference in seconds, check every 14 days
constexpr double expiration = 60 * 60 * 24 * 14;
return std::difftime(std::time(nullptr), last_checked) > expiration;
}
/***************
* MSubdirData *
***************/
/***********************
* SubdirIndexLoader *
***********************/
namespace
{
@ -330,14 +332,14 @@ namespace mamba
using file_time_point = fs::file_time_type::clock::time_point;
template <typename T>
std::vector<T> without_duplicates(std::vector<T>&& values)
auto without_duplicates(std::vector<T>&& values) -> std::vector<T>
{
const auto end_it = std::unique(values.begin(), values.end());
values.erase(end_it, values.end());
return values;
}
file_duration get_cache_age(const fs::u8path& cache_file, const file_time_point& ref)
auto get_cache_age(const fs::u8path& cache_file, const file_time_point& ref) -> file_duration
{
try
{
@ -352,37 +354,38 @@ namespace mamba
}
}
bool is_valid(const file_duration& age)
auto is_valid(const file_duration& age) -> bool
{
return age != file_duration::max();
}
int get_max_age(const std::string& cache_control, int local_repodata_ttl)
[[nodiscard]] auto get_cache_control_max_age(const std::string& cache_control)
-> std::optional<std::size_t>
{
int max_age = local_repodata_ttl;
if (local_repodata_ttl == 1)
static const std::regex max_age_re("max-age=(\\d+)");
std::smatch max_age_match;
const bool matches = std::regex_search(cache_control, max_age_match, max_age_re);
if (!matches)
{
static std::regex max_age_re("max-age=(\\d+)");
std::smatch max_age_match;
bool matches = std::regex_search(cache_control, max_age_match, max_age_re);
if (!matches)
{
max_age = 0;
}
else
{
max_age = std::stoi(max_age_match[1]);
}
return std::nullopt;
}
return max_age;
std::size_t max_age = 0;
const auto& match = max_age_match[1].str();
auto [_, ec] = std::from_chars(match.data(), match.data() + match.size(), max_age);
if (ec == std::errc())
{
return { max_age };
}
return std::nullopt;
}
fs::u8path get_cache_dir(const fs::u8path& cache_path)
auto get_cache_dir(const fs::u8path& cache_path) -> fs::u8path
{
return cache_path / "cache";
}
const fs::u8path& replace_file(const fs::u8path& old_file, const fs::u8path& new_file)
auto replace_file(const fs::u8path& old_file, const fs::u8path& new_file) -> const fs::u8path&
{
if (fs::is_regular_file(old_file))
{
@ -400,14 +403,13 @@ namespace mamba
}
expected_t<SubdirData> SubdirData::create(
auto SubdirIndexLoader::create(
const SubdirParams& params,
ChannelContext& channel_context,
specs::Channel channel,
specs::DynamicPlatform platform,
MultiPackageCache& caches,
std::string repodata_filename
)
) -> expected_t<SubdirIndexLoader>
{
if (channel.is_package())
{
@ -420,9 +422,8 @@ namespace mamba
auto name = get_name(channel.id(), platform);
try
{
return SubdirData(
return SubdirIndexLoader(
params,
channel_context,
std::move(channel),
std::move(platform),
caches,
@ -442,88 +443,95 @@ namespace mamba
}
}
bool SubdirData::is_noarch() const
auto SubdirIndexLoader::is_noarch() const -> bool
{
return specs::platform_is_noarch(m_platform);
}
auto SubdirData::is_local() const -> bool
auto SubdirIndexLoader::is_local() const -> bool
{
return (channel().mirror_urls().size() == 1u) && (channel().url().scheme() == "file");
}
auto SubdirData::channel() const -> const specs::Channel&
auto SubdirIndexLoader::channel() const -> const specs::Channel&
{
return m_channel;
}
auto SubdirData::caching_is_forbidden() const -> bool
auto SubdirIndexLoader::caching_is_forbidden() const -> bool
{
// The only condition yet
return is_local();
}
bool SubdirData::valid_cache_found() const
auto SubdirIndexLoader::valid_cache_found() const -> bool
{
return m_valid_cache_found;
}
void SubdirData::clear_cache_files()
void SubdirIndexLoader::clear_valid_cache_files()
{
if (fs::is_regular_file(m_json_filename))
if (auto json_path = valid_json_cache_path_unchecked(); fs::is_regular_file(json_path))
{
fs::remove(m_json_filename);
fs::remove(json_path);
m_json_cache_valid = false;
}
if (fs::is_regular_file(m_solv_filename))
if (auto state_path = valid_state_file_path_unchecked(); fs::is_regular_file(state_path))
{
fs::remove(m_solv_filename);
fs::remove(state_path);
}
if (auto solv_path = valid_libsolv_cache_path_unchecked(); fs::is_regular_file(solv_path))
{
fs::remove(solv_path);
m_solv_cache_valid = false;
}
m_valid_cache_found = false;
}
std::string SubdirData::name() const
auto SubdirIndexLoader::name() const -> std::string
{
return get_name(channel_id(), m_platform);
}
const std::string& SubdirData::channel_id() const
auto SubdirIndexLoader::channel_id() const -> const std::string&
{
return m_channel.id();
}
const specs::DynamicPlatform& SubdirData::platform() const
auto SubdirIndexLoader::platform() const -> const specs::DynamicPlatform&
{
return m_platform;
}
const SubdirMetadata& SubdirData::metadata() const
auto SubdirIndexLoader::metadata() const -> const SubdirMetadata&
{
return m_metadata;
}
expected_t<fs::u8path> SubdirData::valid_libsolv_cache_path() const
auto SubdirIndexLoader::valid_libsolv_cache_path() const -> expected_t<fs::u8path>
{
if (m_json_cache_valid && m_solv_cache_valid)
{
return (get_cache_dir(m_valid_cache_path) / m_solv_filename).string();
return { valid_libsolv_cache_path_unchecked() };
}
return make_unexpected("Cache not loaded", mamba_error_code::cache_not_loaded);
}
fs::u8path SubdirData::writable_libsolv_cache_path() const
auto SubdirIndexLoader::writable_libsolv_cache_path() const -> fs::u8path
{
return m_writable_pkgs_dir / "cache" / m_solv_filename;
}
expected_t<fs::u8path> SubdirData::valid_json_cache_path() const
auto SubdirIndexLoader::valid_json_cache_path() const -> expected_t<fs::u8path>
{
if (m_json_cache_valid)
{
return (get_cache_dir(m_valid_cache_path) / m_json_filename).string();
return { valid_json_cache_path_unchecked() };
}
return make_unexpected("Cache not loaded", mamba_error_code::cache_not_loaded);
}
auto SubdirData::download_requests(
auto SubdirIndexLoader::download_requests(
download::MultiRequest requests,
const specs::AuthenticationDataBase& auth_info,
const download::mirror_map& mirrors,
@ -534,7 +542,7 @@ namespace mamba
{
try
{
download::download(
auto results = download::download(
std::move(requests),
mirrors,
remote_fetch_params,
@ -542,6 +550,15 @@ namespace mamba
download_options,
monitor
);
// TODO: This is not the best handling, but we also want to be robust in the case of
// missing subdirs (e.g. local path as a `noarch` but no `linux-64`).
for (auto& result : results)
{
if (!result.has_value())
{
LOG_WARNING << "Failed to load subdir: " << result.error().message;
}
}
}
catch (const std::runtime_error& e)
{
@ -555,17 +572,14 @@ namespace mamba
return expected_t<void>();
}
SubdirData::SubdirData(
SubdirIndexLoader::SubdirIndexLoader(
const SubdirParams& params,
ChannelContext& channel_context,
specs::Channel channel,
std::string platform,
MultiPackageCache& caches,
std::string repodata_filename
)
: m_channel(std::move(channel))
, m_valid_cache_path("")
, m_expired_cache_path("")
, m_writable_pkgs_dir(caches.first_writable_path())
, m_platform(std::move(platform))
, m_repodata_filename(std::move(repodata_filename))
@ -573,48 +587,57 @@ namespace mamba
, m_solv_filename(m_json_filename.substr(0, m_json_filename.size() - 4) + "solv")
{
assert(!this->channel().is_package());
load(caches, channel_context, params, this->channel());
load(caches, params);
}
std::string SubdirData::repodata_url_path() const
auto SubdirIndexLoader::repodata_url_path() const -> std::string
{
return util::url_concat(m_platform, "/", m_repodata_filename);
}
specs::CondaURL SubdirData::repodata_url() const
auto SubdirIndexLoader::valid_json_cache_path_unchecked() const -> fs::u8path
{
return get_cache_dir(m_valid_cache_path) / m_json_filename;
}
auto SubdirIndexLoader::valid_state_file_path_unchecked() const -> fs::u8path
{
auto state_file = valid_json_cache_path_unchecked();
state_file.replace_extension(".state.json");
return state_file;
}
auto SubdirIndexLoader::valid_libsolv_cache_path_unchecked() const -> fs::u8path
{
return get_cache_dir(m_valid_cache_path) / m_solv_filename;
}
auto SubdirIndexLoader::repodata_url() const -> specs::CondaURL
{
return channel().platform_url(m_platform) / m_repodata_filename;
}
void SubdirData::load(
const MultiPackageCache& caches,
ChannelContext& channel_context,
const SubdirParams& params,
const specs::Channel& channel
)
void SubdirIndexLoader::load(const MultiPackageCache& caches, const SubdirParams& params)
{
// For local channel subdirs, we still go through the downloaders
if (!caching_is_forbidden())
{
load_cache(caches, params);
}
if (m_valid_cache_found)
if (params.repodata_force_use_zst)
{
Console::stream() << fmt::format("{:<50} {:>20}", name(), std::string("Using cache"));
m_metadata.set_zst(true);
}
else
LOG_INFO << "Valid cache found for '" << name() << "': " << valid_cache_found();
if (!valid_cache_found() && m_expired_cache_path.has_value())
{
LOG_INFO << "No valid cache found";
if (!m_expired_cache_path.empty())
{
LOG_INFO << "Expired cache (or invalid mod/etag headers) found at '"
<< m_expired_cache_path.string() << "'";
}
update_metadata_zst(channel_context, params, channel);
LOG_INFO << "Expired cache (or invalid mod/etag headers) found at '"
<< m_expired_cache_path.value() << "'";
}
}
void SubdirData::load_cache(const MultiPackageCache& caches, const SubdirParams& params)
void SubdirIndexLoader::load_cache(const MultiPackageCache& caches, const SubdirParams& params)
{
LOG_INFO << "Searching index cache file for repo '" << name() << "'";
file_time_point now = fs::file_time_type::clock::now();
@ -623,14 +646,15 @@ namespace mamba
for (const fs::u8path& cache_path : cache_paths)
{
const auto index_cache_path = get_cache_dir(cache_path);
// TODO: rewrite this with pipe chains of ranges
fs::u8path json_file = cache_path / "cache" / m_json_filename;
fs::u8path json_file = index_cache_path / m_json_filename;
if (!fs::is_regular_file(json_file))
{
continue;
}
auto lock = LockFile(cache_path / "cache");
auto lock = LockFile(index_cache_path);
file_duration cache_age = get_cache_age(json_file, now);
if (!is_valid(cache_age))
{
@ -645,14 +669,25 @@ namespace mamba
}
m_metadata = std::move(metadata_temp.value());
const int max_age = get_max_age(
m_metadata.cache_control(),
static_cast<int>(params.local_repodata_ttl)
);
// TODO(C++23): Use std::optional::and_then
const std::size_t max_age = [&]()
{
static constexpr std::size_t max_age_default = 60 * 60;
if (params.local_repodata_ttl_s)
{
return params.local_repodata_ttl_s.value();
}
if (auto control_max_age = get_cache_control_max_age(m_metadata.cache_control()))
{
return control_max_age.value();
}
return max_age_default;
}();
const auto cache_age_seconds = std::chrono::duration_cast<std::chrono::seconds>(cache_age)
.count();
if ((max_age > cache_age_seconds || params.offline || params.use_index_cache))
if (util::cmp_less(cache_age_seconds, max_age) || params.offline)
{
// valid json cache found
if (!m_valid_cache_found)
@ -666,7 +701,7 @@ namespace mamba
}
// check libsolv cache
fs::u8path solv_file = cache_path / "cache" / m_solv_filename;
fs::u8path solv_file = index_cache_path / m_solv_filename;
file_duration solv_age = get_cache_age(solv_file, now);
if (is_valid(solv_age) && solv_age <= cache_age)
@ -684,7 +719,7 @@ namespace mamba
}
else
{
if (m_expired_cache_path.empty())
if (!m_expired_cache_path.has_value())
{
m_expired_cache_path = cache_path;
}
@ -693,23 +728,12 @@ namespace mamba
}
}
void SubdirData::update_metadata_zst(
ChannelContext& channel_context,
const SubdirParams& params,
const specs::Channel& channel
)
{
if (!params.offline || caching_is_forbidden())
{
m_metadata.set_zst(m_metadata.has_up_to_date_zst() || channel_context.has_zst(channel));
}
}
download::MultiRequest SubdirData::build_check_requests(const SubdirParams& params)
auto SubdirIndexLoader::build_check_requests(const SubdirDownloadParams& params)
-> download::MultiRequest
{
download::MultiRequest request;
if ((!params.offline || caching_is_forbidden()) && params.repodata_use_zst
if ((!params.offline || caching_is_forbidden()) && params.repodata_check_zst
&& !m_metadata.has_up_to_date_zst())
{
request.push_back(download::Request(
@ -746,7 +770,7 @@ namespace mamba
return request;
}
auto SubdirData::build_index_request(const SubdirParams& params)
auto SubdirIndexLoader::build_index_request(const SubdirDownloadParams& params)
-> std::optional<download::Request>
{
if (params.offline && !caching_is_forbidden())
@ -814,18 +838,20 @@ namespace mamba
return { std::move(request) };
}
expected_t<void> SubdirData::use_existing_cache()
auto SubdirIndexLoader::use_existing_cache() -> expected_t<void>
{
LOG_INFO << "Cache is still valid";
fs::u8path json_file = m_expired_cache_path / "cache" / m_json_filename;
fs::u8path solv_file = m_expired_cache_path / "cache" / m_solv_filename;
assert(m_expired_cache_path.has_value());
fs::u8path json_file = m_expired_cache_path.value() / "cache" / m_json_filename;
fs::u8path solv_file = m_expired_cache_path.value() / "cache" / m_solv_filename;
if (path::is_writable(json_file)
&& (!fs::is_regular_file(solv_file) || path::is_writable(solv_file)))
{
LOG_DEBUG << "Refreshing cache files ages";
m_valid_cache_path = m_expired_cache_path;
m_valid_cache_path = m_expired_cache_path.value();
}
else
{
@ -838,7 +864,7 @@ namespace mamba
);
}
LOG_DEBUG << "Copying repodata cache files from '" << m_expired_cache_path.string()
LOG_DEBUG << "Copying repodata cache files from '" << m_expired_cache_path.value()
<< "' to '" << m_writable_pkgs_dir.string() << "'";
fs::u8path writable_cache_dir = get_cache_dir(m_writable_pkgs_dir);
auto lock = LockFile(writable_cache_dir);
@ -861,8 +887,9 @@ namespace mamba
return expected_t<void>();
}
expected_t<void>
SubdirData::finalize_transfer(SubdirMetadata::HttpMetadata http_data, const fs::u8path& artifact)
auto
SubdirIndexLoader::finalize_transfer(SubdirMetadata::HttpMetadata http_data, const fs::u8path& artifact)
-> expected_t<void>
{
if (m_writable_pkgs_dir.empty())
{
@ -907,7 +934,8 @@ namespace mamba
return expected_t<void>();
}
void SubdirData::refresh_last_write_time(const fs::u8path& json_file, const fs::u8path& solv_file)
void
SubdirIndexLoader::refresh_last_write_time(const fs::u8path& json_file, const fs::u8path& solv_file)
{
const auto now = fs::file_time_type::clock::now();
@ -934,7 +962,7 @@ namespace mamba
m_metadata.write_state_file(state_file);
}
std::string cache_name_from_url(std::string url)
auto cache_name_from_url(std::string url) -> std::string
{
if (url.empty() || (url.back() != '/' && !util::ends_with(url, ".json")))
{
@ -950,12 +978,12 @@ namespace mamba
return util::Md5Hasher().str_hex_str(url).substr(0, 8u);
}
std::string cache_filename_from_url(std::string url)
auto cache_filename_from_url(std::string url) -> std::string
{
return cache_name_from_url(std::move(url)) + ".json";
}
std::string create_cache_dir(const fs::u8path& cache_path)
auto create_cache_dir(const fs::u8path& cache_path) -> std::string
{
const auto cache_dir = cache_path / "cache";
fs::create_directories(cache_dir);

7
libmamba/tests/CMakeLists.txt
View File

@ -89,6 +89,7 @@ set(
src/core/test_pinning.cpp
src/core/test_progress_bar.cpp
src/core/test_shell_init.cpp
src/core/test_subdir_index.cpp
src/core/test_tasksync.cpp
src/core/test_thread_utils.cpp
src/core/test_transaction_context.cpp
@ -135,8 +136,10 @@ add_dependencies(test_libmamba test_libmamba_data)
target_compile_definitions(
test_libmamba
PRIVATE MAMBA_TEST_DATA_DIR="${CMAKE_CURRENT_BINARY_DIR}/data"
MAMBA_TEST_LOCK_EXE="$<TARGET_FILE:testing_libmamba_lock>"
PRIVATE
MAMBA_TEST_DATA_DIR="${CMAKE_CURRENT_BINARY_DIR}/data"
MAMBA_REPO_DIR="${CMAKE_SOURCE_DIR}"
MAMBA_TEST_LOCK_EXE="$<TARGET_FILE:testing_libmamba_lock>"
)
target_compile_features(test_libmamba PUBLIC cxx_std_17)

5
libmamba/tests/include/mambatests.hpp
View File

@ -24,6 +24,11 @@ namespace mambatests
#endif
inline static const mamba::fs::u8path test_data_dir = MAMBA_TEST_DATA_DIR;
#ifndef MAMBA_REPO_DIR
#error "MAMBA_REPO_DIR must be defined pointing to test data"
#endif
inline static const mamba::fs::u8path repo_dir = MAMBA_REPO_DIR;
#ifndef MAMBA_TEST_LOCK_EXE
#error "MAMBA_TEST_LOCK_EXE must be defined pointing to testing_libmamba_lock"
#endif

2
libmamba/tests/src/core/test_cpp.cpp
View File

@ -16,7 +16,7 @@
#include "mamba/core/history.hpp"
#include "mamba/core/link.hpp"
#include "mamba/core/output.hpp"
#include "mamba/core/subdirdata.hpp"
#include "mamba/core/subdir_index.hpp"
#include "mamba/util/build.hpp"
#include "mamba/util/path_manip.hpp"

2
libmamba/tests/src/core/test_filesystem.cpp
View File

@ -8,7 +8,7 @@
#include <catch2/catch_all.hpp>
#include "mamba/core/subdirdata.hpp"
#include "mamba/core/subdir_index.hpp"
#include "mamba/core/util.hpp"
#include "mamba/core/util_scope.hpp"
#include "mamba/fs/filesystem.hpp"

233
libmamba/tests/src/core/test_subdir_index.cpp Normal file
View File

@ -0,0 +1,233 @@
// Copyright (c) 2023, QuantStack and Mamba Contributors
//
// Distributed under the terms of the BSD 3-Clause License.
//
// The full license is in the file LICENSE, distributed with this software.
#include <array>
#include <fstream>
#include <sstream>
#include <catch2/catch_all.hpp>
#include "mamba/core/channel_context.hpp"
#include "mamba/core/context.hpp"
#include "mamba/core/package_cache.hpp"
#include "mamba/core/subdir_index.hpp"
#include "mamba/core/util.hpp"
#include "mamba/util/string.hpp"
#include "mambatests.hpp"
using namespace mamba;
namespace
{
[[nodiscard]] auto is_in_directory(const fs::u8path& dir, const fs::u8path& file) -> bool
{
auto abs_dir = fs::absolute(dir).lexically_normal();
auto abs_file = fs::absolute(file).lexically_normal();
return abs_file.parent_path() == abs_dir;
}
[[nodiscard]] auto file_to_string(const fs::u8path& filename) -> std::string
{
std::ifstream file(filename.string());
std::ostringstream ss;
ss << file.rdbuf();
return ss.str();
}
[[nodiscard]] auto make_simple_channel(std::string_view chan) -> specs::Channel
{
const auto resolve_params = ChannelContext::ChannelResolveParams{
{ "linux-64", "osx-64", "noarch" },
specs::CondaURL::parse("https://conda.anaconda.org").value()
};
return specs::Channel::resolve(specs::UnresolvedChannel::parse(chan).value(), resolve_params)
.value()
.front();
}
}
TEST_CASE("SubdirIndexLoader", "[mamba::core][mamba::core::SubdirIndexLoader]")
{
const auto resolve_params = ChannelContext::ChannelResolveParams{
{ "linux-64", "osx-64", "noarch" },
specs::CondaURL::parse("https://conda.anaconda.org").value()
};
const auto qs_channel = make_simple_channel("quantstack");
const auto local_repo_path = mambatests::repo_dir / "micromamba/test-server/repo/";
const auto local_channel = make_simple_channel(local_repo_path.string());
auto mirrors = download::mirror_map();
for (const auto& chan : { qs_channel, local_channel })
{
mirrors.add_unique_mirror(chan.id(), download::make_mirror(chan.url().str()));
}
SECTION("Create a subdir loader")
{
constexpr auto platform = "mamba-128";
constexpr auto repodata_filename = "foo.json";
const auto tmp_dir = TemporaryDirectory();
auto caches = MultiPackageCache({ tmp_dir.path() }, ValidationParams{});
auto subdir = SubdirIndexLoader::create({}, qs_channel, platform, caches, repodata_filename);
REQUIRE(subdir.has_value());
CHECK_FALSE(subdir->is_noarch());
CHECK_FALSE(subdir->is_local());
CHECK(subdir->channel() == qs_channel);
CHECK(subdir->name() == "quantstack/mamba-128");
CHECK(subdir->channel_id() == "quantstack");
CHECK(subdir->platform() == platform);
CHECK(
subdir->repodata_url()
== specs::CondaURL::parse("https://conda.anaconda.org/quantstack/mamba-128/foo.json").value()
);
const auto& metadata = subdir->metadata();
CHECK(metadata.url() == "");
CHECK(metadata.etag() == "");
CHECK_FALSE(subdir->valid_cache_found());
CHECK_FALSE(subdir->valid_libsolv_cache_path().has_value());
CHECK_FALSE(subdir->valid_json_cache_path().has_value());
}
SECTION("Download indexes")
{
const auto tmp_dir = TemporaryDirectory();
auto caches = MultiPackageCache({ tmp_dir.path() }, ValidationParams{});
auto subdirs = std::array{
SubdirIndexLoader::create({}, qs_channel, "linux-64", caches).value(),
SubdirIndexLoader::create({}, qs_channel, "noarch", caches).value(),
};
auto result = SubdirIndexLoader::download_required_indexes(subdirs, {}, {}, mirrors, {}, {});
REQUIRE(result.has_value());
const auto cache_dir = tmp_dir.path() / "cache";
for (const auto& subdir : subdirs)
{
CHECK(subdir.valid_cache_found());
CHECK(is_in_directory(cache_dir, subdir.valid_json_cache_path().value()));
CHECK(util::contains(file_to_string(subdir.valid_json_cache_path().value()), "packages"));
CHECK_FALSE(subdir.valid_libsolv_cache_path().has_value());
CHECK(is_in_directory(cache_dir, subdir.writable_libsolv_cache_path()));
}
SECTION("And clear them")
{
for (auto& subdir : subdirs)
{
subdir.clear_valid_cache_files();
CHECK_FALSE(subdir.valid_cache_found());
CHECK_FALSE(subdir.valid_json_cache_path().has_value());
CHECK_FALSE(subdir.valid_libsolv_cache_path().has_value());
}
CHECK(fs::is_empty(cache_dir));
}
}
SECTION("No download offline")
{
const auto tmp_dir = TemporaryDirectory();
auto caches = MultiPackageCache({ tmp_dir.path() }, ValidationParams{});
const auto params = SubdirParams{
/* .local_repodata_ttl */ 1000000,
/* .offline */ true,
};
auto subdirs = std::array{
SubdirIndexLoader::create(params, qs_channel, "linux-64", caches).value(),
SubdirIndexLoader::create(params, qs_channel, "noarch", caches).value(),
};
const auto download_params = SubdirDownloadParams{
/* .offline */ true,
};
auto result = SubdirIndexLoader::download_required_indexes(
subdirs,
download_params,
{},
mirrors,
{},
{}
);
REQUIRE(result.has_value());
const auto cache_dir = tmp_dir.path() / "cache";
for (const auto& subdir : subdirs)
{
CHECK_FALSE(subdir.valid_cache_found());
}
}
SECTION("Local noarch-only repo offline")
{
const auto tmp_dir = TemporaryDirectory();
auto caches = MultiPackageCache({ tmp_dir.path() }, ValidationParams{});
const auto params = SubdirParams{
/* .local_repodata_ttl */ 1000000,
/* .offline */ true,
};
auto subdirs = std::array{
SubdirIndexLoader::create(params, local_channel, "linux-64", caches).value(),
SubdirIndexLoader::create(params, local_channel, "noarch", caches).value(),
};
auto result = SubdirIndexLoader::download_required_indexes(subdirs, {}, {}, mirrors, {}, {});
REQUIRE(result.has_value());
CHECK_FALSE(subdirs[0].valid_cache_found());
CHECK(subdirs[1].valid_cache_found());
CHECK(subdirs[1].valid_json_cache_path().has_value());
}
SECTION("Download indexes repodata ttl")
{
const auto tmp_dir = TemporaryDirectory();
auto caches = MultiPackageCache({ tmp_dir.path() }, ValidationParams{});
const auto params = SubdirParams{
/* .local_repodata_ttl */ 0,
};
auto subdirs = std::array{
SubdirIndexLoader::create(params, qs_channel, "linux-64", caches).value(),
SubdirIndexLoader::create(params, qs_channel, "noarch", caches).value(),
};
auto result = SubdirIndexLoader::download_required_indexes(subdirs, {}, {}, mirrors, {}, {});
REQUIRE(result.has_value());
const auto cache_dir = tmp_dir.path() / "cache";
for (const auto& subdir : subdirs)
{
CHECK(subdir.valid_cache_found());
}
SECTION("Reloading subdir are expired")
{
auto expired_subdirs = std::array{
SubdirIndexLoader::create(params, qs_channel, "linux-64", caches).value(),
SubdirIndexLoader::create(params, qs_channel, "noarch", caches).value(),
};
for (const auto& subdir : expired_subdirs)
{
CHECK_FALSE(subdir.valid_cache_found());
}
}
}
}

16
libmamba/tests/src/solver/test_problems_graph.cpp
View File

@ -17,7 +17,7 @@
#include "mamba/core/channel_context.hpp"
#include "mamba/core/package_database_loader.hpp"
#include "mamba/core/prefix_data.hpp"
#include "mamba/core/subdirdata.hpp"
#include "mamba/core/subdir_index.hpp"
#include "mamba/core/util.hpp"
#include "mamba/fs/filesystem.hpp"
#include "mamba/solver/libsolv/database.hpp"
@ -334,7 +334,7 @@ namespace
std::vector<std::string>&& channels
)
{
auto sub_dirs = std::vector<SubdirData>();
auto sub_dirs = std::vector<SubdirIndexLoader>();
for (const auto& location : channels)
{
for (const auto& chan : channel_context.make_channel(location))
@ -342,22 +342,16 @@ namespace
create_mirrors(ctx, chan);
for (const auto& platform : chan.platforms())
{
auto sub_dir = SubdirData::create(
ctx.subdir_params(),
channel_context,
chan,
platform,
cache
)
auto sub_dir = SubdirIndexLoader::create(ctx.subdir_params(), chan, platform, cache)
.value();
sub_dirs.push_back(std::move(sub_dir));
}
}
}
const auto result = SubdirData::download_required_indexes(
const auto result = SubdirIndexLoader::download_required_indexes(
sub_dirs,
mambatests::context().subdir_params(),
mambatests::context().subdir_download_params(),
mambatests::context().authentication_info(),
mambatests::context().mirrors,
mambatests::context().download_options(),

30
libmambapy/src/libmambapy/bindings/expected_caster.hpp
View File

@ -59,6 +59,36 @@ namespace PYBIND11_NAMESPACE
PYBIND11_TYPE_CASTER(value_type, detail::concat(make_caster<T>::name, make_caster<E>::name));
};
template <typename E>
struct type_caster<tl::expected<void, E>>
{
using value_type = std::nullptr_t;
auto load(handle, bool) -> bool
{
return true;
}
template <typename Expected>
static auto cast(Expected&& src, return_value_policy, handle) -> handle
{
if (src)
{
return none();
}
else
{
// If we use ``expected`` without exception in our API, we need to convert them
// to an exception before throwing it in PyBind11 code.
// This could be done with partial specialization of this ``type_caster``.
static_assert(std::is_base_of_v<std::exception, E>);
throw std::forward<Expected>(src).error();
}
}
PYBIND11_TYPE_CASTER(value_type, const_name("None"));
};
}
}
#endif

207
libmambapy/src/libmambapy/bindings/legacy.cpp
View File

@ -27,7 +27,7 @@
#include "mamba/core/package_handling.hpp"
#include "mamba/core/prefix_data.hpp"
#include "mamba/core/query.hpp"
#include "mamba/core/subdirdata.hpp"
#include "mamba/core/subdir_index.hpp"
#include "mamba/core/transaction.hpp"
#include "mamba/core/util_os.hpp"
#include "mamba/core/virtual_packages.hpp"
@ -133,13 +133,15 @@ namespace mambapy
// replicate the move semantics in Python, we encapsulate the creation
// and the storage of MSubdirData objects in this class, to avoid
// potential dangling references in Python.
//
// Deprecated, replaced by SubdirIndexLoader in 2.3.0
class SubdirIndex
{
public:
struct Entry
{
mamba::SubdirData* p_subdirdata = nullptr;
mamba::SubdirIndexLoader* p_subdirdata = nullptr;
std::string m_platform = "";
const mamba::specs::Channel* p_channel = nullptr;
std::string m_url = "";
@ -161,8 +163,10 @@ namespace mambapy
)
{
using namespace mamba;
auto subdir_params = ctx.subdir_params();
subdir_params.repodata_force_use_zst = channel_context.has_zst(channel);
m_subdirs.push_back(extract(
SubdirData::create(ctx.subdir_params(), channel_context, channel, platform, caches, repodata_fn)
SubdirIndexLoader::create(subdir_params, channel, platform, caches, repodata_fn)
));
m_entries.push_back({ nullptr, platform, &channel, url });
for (size_t i = 0; i < m_subdirs.size(); ++i)
@ -177,13 +181,13 @@ namespace mambapy
// TODO: expose SubdirDataMonitor to libmambapy and remove this
// logic
expected_t<void> download_res;
if (SubdirDataMonitor::can_monitor(ctx))
if (SubdirIndexMonitor::can_monitor(ctx))
{
SubdirDataMonitor check_monitor({ true, true });
SubdirDataMonitor index_monitor;
download_res = SubdirData::download_required_indexes(
SubdirIndexMonitor check_monitor({ true, true });
SubdirIndexMonitor index_monitor;
download_res = SubdirIndexLoader::download_required_indexes(
m_subdirs,
ctx.subdir_params(),
ctx.subdir_download_params(),
ctx.authentication_info(),
ctx.mirrors,
ctx.download_options(),
@ -194,9 +198,9 @@ namespace mambapy
}
else
{
download_res = SubdirData::download_required_indexes(
download_res = SubdirIndexLoader::download_required_indexes(
m_subdirs,
ctx.subdir_params(),
ctx.subdir_download_params(),
ctx.authentication_info(),
ctx.mirrors,
ctx.download_options(),
@ -228,7 +232,7 @@ namespace mambapy
private:
std::vector<mamba::SubdirData> m_subdirs;
std::vector<mamba::SubdirIndexLoader> m_subdirs;
entry_list m_entries;
};
}
@ -433,6 +437,30 @@ bind_submodule_impl(pybind11::module_ m)
py::add_ostream_redirect(m, "ostream_redirect");
py::class_<download::RemoteFetchParams>(m, "RemoteFetchParams")
.def(py::init<>())
.def_readwrite("ssl_verify", &download::RemoteFetchParams::ssl_verify)
.def_readwrite("max_retries", &download::RemoteFetchParams::max_retries)
.def_readwrite("retry_timeout", &download::RemoteFetchParams::retry_timeout)
.def_readwrite("retry_backoff", &download::RemoteFetchParams::retry_backoff)
.def_readwrite("user_agent", &download::RemoteFetchParams::user_agent)
// .def_readwrite("read_timeout_secs", &Context::RemoteFetchParams::read_timeout_secs)
.def_readwrite("proxy_servers", &download::RemoteFetchParams::proxy_servers)
.def_readwrite("connect_timeout_secs", &download::RemoteFetchParams::connect_timeout_secs);
py::class_<download::Options>(m, "DownloadOptions")
.def(py::init<>())
.def_readwrite("download_threads", &download::Options::download_threads)
.def_readwrite("fail_fast", &download::Options::fail_fast)
.def_readwrite("sort", &download::Options::sort)
.def_readwrite("verbose", &download::Options::verbose);
py::class_<download::mirror_map>(m, "MirrorMap")
.def(py::init<>())
.def("has_mirrors", &download::mirror_map::has_mirrors, py::arg("mirror_name"))
.def("__contains__", &download::mirror_map::has_mirrors)
.def("__len__", &download::mirror_map::size);
m.def(
"load_subdir_in_database",
&load_subdir_in_database,
@ -526,25 +554,123 @@ bind_submodule_impl(pybind11::module_ m)
.def_static("whoneeds", &Query::whoneeds)
.def_static("depends", &Query::depends);
py::class_<SubdirData>(m, "SubdirData")
py::class_<SubdirParams>(m, "SubdirParams")
.def_readwrite("local_repodata_ttl_s", &SubdirParams::local_repodata_ttl_s)
.def_readwrite("offline", &SubdirParams::offline)
.def_readwrite("repodata_force_use_zst", &SubdirParams::repodata_force_use_zst);
py::class_<SubdirDownloadParams>(m, "SubdirDownloadParams")
.def_readwrite("offline", &SubdirDownloadParams::offline)
.def_readwrite("repodata_check_zst", &SubdirDownloadParams::repodata_check_zst);
auto subdir_metadata = py::class_<SubdirMetadata>(m, "SubdirMetadata");
py::class_<SubdirMetadata::HttpMetadata>(subdir_metadata, "HttpMetadata")
.def_readwrite("url", &SubdirMetadata::HttpMetadata::url)
.def_readwrite("etag", &SubdirMetadata::HttpMetadata::etag)
.def_readwrite("last_modified", &SubdirMetadata::HttpMetadata::last_modified)
.def_readwrite("cache_control", &SubdirMetadata::HttpMetadata::cache_control);
subdir_metadata.def_static("read_state_file", &SubdirMetadata::read_state_file)
.def_static("read_from_repodata_json", &SubdirMetadata::read_from_repodata_json)
.def_static("read", &SubdirMetadata::read)
.def("is_valid_metadata", &SubdirMetadata::is_valid_metadata)
.def("url", &SubdirMetadata::url)
.def("etag", &SubdirMetadata::etag)
.def("last_modified", &SubdirMetadata::last_modified)
.def("cache_control", &SubdirMetadata::cache_control)
.def("has_up_to_date_zst", &SubdirMetadata::has_up_to_date_zst)
.def("set_http_metadata", &SubdirMetadata::set_http_metadata)
.def("set_zst", &SubdirMetadata::set_zst)
.def("store_file_metadata", &SubdirMetadata::store_file_metadata)
.def("write_state_file", &SubdirMetadata::write_state_file);
py::class_<SubdirIndexLoader>(m, "SubdirIndexLoader")
.def_static(
"create",
SubdirIndexLoader::create,
py::arg("params"),
py::arg("channel"),
py::arg("platform"),
py::arg("caches"),
py::arg("repodata_filename") = "repodata.json"
)
.def_static(
"download_required_indexes",
[](py::iterable py_subdirs,
const SubdirDownloadParams& subdir_download_params,
const specs::AuthenticationDataBase& auth_info,
const download::mirror_map& mirrors,
const download::Options& download_options,
const download::RemoteFetchParams& remote_fetch_params)
{
// TODO(C++23): Pass range to SubdirIndexLoader::create
auto subdirs = std::vector<SubdirIndexLoader*>();
subdirs.reserve(py::len_hint(py_subdirs));
for (py::handle item : py_subdirs)
{
subdirs.push_back(py::cast<SubdirIndexLoader*>(item));
}
return SubdirIndexLoader::download_required_indexes(
subdirs,
subdir_download_params,
auth_info,
mirrors,
download_options,
remote_fetch_params
);
},
py::arg("subdir_indices"),
py::arg("subdir_params"),
py::arg("auth_info"),
py::arg("mirrors"),
py::arg("download_options"),
py::arg("remote_fetch_params")
)
.def("is_noarch", &SubdirIndexLoader::is_noarch)
.def("is_local", &SubdirIndexLoader::is_local)
.def("channel", &SubdirIndexLoader::channel)
.def("name", &SubdirIndexLoader::name)
.def("channel_id", &SubdirIndexLoader::channel_id)
.def("platform", &SubdirIndexLoader::platform)
.def("metadata", &SubdirIndexLoader::metadata)
.def("repodata_url", &SubdirIndexLoader::repodata_url)
.def("caching_is_forbidden", &SubdirIndexLoader::caching_is_forbidden)
.def("valid_cache_found", &SubdirIndexLoader::valid_cache_found)
.def("valid_libsolv_cache_path", &SubdirIndexLoader::valid_libsolv_cache_path)
.def("writable_libsolv_cache_path", &SubdirIndexLoader::writable_libsolv_cache_path)
.def("valid_json_cache_path", &SubdirIndexLoader::valid_json_cache_path)
.def("clear_valid_cache_files", &SubdirIndexLoader::clear_valid_cache_files);
// Deprecated, replaced by SubdirIndexLoader in 2.3.0
struct SubdirDataMigrator
{
mamba::SubdirIndexLoader* p_subdir_index;
};
// Deprecated, replaced by SubdirIndexLoader in 2.3.0
py::class_<SubdirDataMigrator>(m, "SubdirData")
.def(
"create_repo",
[](SubdirData& self, Context& context, solver::libsolv::Database& database
[](SubdirDataMigrator& self, Context& context, solver::libsolv::Database& database
) -> solver::libsolv::RepoInfo
{
deprecated("Use libmambapy.load_subdir_in_database instead", "2.0");
return extract(load_subdir_in_database(context, database, self));
return extract(load_subdir_in_database(context, database, *self.p_subdir_index));
},
py::arg("context"),
py::arg("db")
)
.def("loaded", &SubdirData::valid_cache_found)
.def(
"loaded",
[](const SubdirDataMigrator& self) { return self.p_subdir_index->valid_cache_found(); }
)
.def(
"valid_solv_cache",
// TODO make a proper well tested type caster for expected types.
[](const SubdirData& self) -> std::optional<fs::u8path>
[](const SubdirDataMigrator& self) -> std::optional<fs::u8path>
{
if (auto f = self.valid_libsolv_cache_path())
if (auto f = self.p_subdir_index->valid_libsolv_cache_path())
{
return { *std::move(f) };
}
@ -553,9 +679,9 @@ bind_submodule_impl(pybind11::module_ m)
)
.def(
"valid_json_cache",
[](const SubdirData& self) -> std::optional<fs::u8path>
[](const SubdirDataMigrator& self) -> std::optional<fs::u8path>
{
if (auto f = self.valid_json_cache_path())
if (auto f = self.p_subdir_index->valid_json_cache_path())
{
return { *std::move(f) };
}
@ -564,17 +690,17 @@ bind_submodule_impl(pybind11::module_ m)
)
.def(
"cache_path",
[](const SubdirData& self) -> std::string
[](const SubdirDataMigrator& self) -> std::string
{
deprecated(
"Use `SubdirData.valid_solv_path` or `SubdirData.valid_json` path instead",
"2.0"
);
if (auto solv_path = self.valid_libsolv_cache_path())
if (auto solv_path = self.p_subdir_index->valid_libsolv_cache_path())
{
return solv_path->string();
}
else if (auto json_path = self.valid_json_cache_path())
else if (auto json_path = self.p_subdir_index->valid_json_cache_path())
{
return json_path->string();
}
@ -585,15 +711,32 @@ bind_submodule_impl(pybind11::module_ m)
using mambapy::SubdirIndex;
using SubdirIndexEntry = SubdirIndex::Entry;
// Deprecated, replaced by SubdirIndexLoader in 2.3.0
py::class_<SubdirIndexEntry>(m, "SubdirIndexEntry")
.def(py::init<>())
.def_readonly("subdir", &SubdirIndexEntry::p_subdirdata, py::return_value_policy::reference)
.def(py::init(
[]()
{
deprecated("Use SubdirIndexLoader", "2.3.0");
return SubdirIndexEntry();
}
))
.def_property_readonly(
"subdir",
[](const SubdirIndexEntry& self) { return SubdirDataMigrator{ self.p_subdirdata }; },
py::return_value_policy::reference
)
.def_readonly("platform", &SubdirIndexEntry::m_platform)
.def_readonly("channel", &SubdirIndexEntry::p_channel, py::return_value_policy::reference)
.def_readonly("url", &SubdirIndexEntry::m_url);
py::class_<SubdirIndex>(m, "SubdirIndex")
.def(py::init<>())
.def(py::init(
[]()
{
deprecated("Use SubdirIndexLoader", "2.3.0");
return SubdirIndex();
}
))
.def(
"create",
[](SubdirIndex& self,
@ -774,16 +917,10 @@ bind_submodule_impl(pybind11::module_ m)
.def("set_verbosity", &Context::set_verbosity)
.def("set_log_level", &Context::set_log_level);
py::class_<download::RemoteFetchParams>(ctx, "RemoteFetchParams")
.def(py::init<>())
.def_readwrite("ssl_verify", &download::RemoteFetchParams::ssl_verify)
.def_readwrite("max_retries", &download::RemoteFetchParams::max_retries)
.def_readwrite("retry_timeout", &download::RemoteFetchParams::retry_timeout)
.def_readwrite("retry_backoff", &download::RemoteFetchParams::retry_backoff)
.def_readwrite("user_agent", &download::RemoteFetchParams::user_agent)
// .def_readwrite("read_timeout_secs", &Context::RemoteFetchParams::read_timeout_secs)
.def_readwrite("proxy_servers", &download::RemoteFetchParams::proxy_servers)
.def_readwrite("connect_timeout_secs", &download::RemoteFetchParams::connect_timeout_secs);
ctx.def_property_readonly_static(
"RemoteFetchParams",
[](py::handle) { return py::type::of<download::RemoteFetchParams>(); }
);
py::class_<Context::OutputParams>(ctx, "OutputParams")
.def(py::init<>())

2
micromamba/src/constructor.cpp
View File

@ -10,7 +10,7 @@
#include "mamba/api/configuration.hpp"
#include "mamba/api/install.hpp"
#include "mamba/core/package_handling.hpp"
#include "mamba/core/subdirdata.hpp"
#include "mamba/core/subdir_index.hpp"
#include "mamba/core/util.hpp"
#include "mamba/util/string.hpp"