folkslinux
/
mamba
mirror of https://github.com/mamba-org/mamba.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove Context from downloaders (#3928)

This commit is contained in:
Antoine Prouvost 2025-05-09 09:58:13 +02:00 committed by GitHub
parent 562f64e8a6
commit 8254680fd7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 287 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
libmamba/include/mamba/core/context.hpp
View File

@ -16,6 +16,7 @@
#include "mamba/core/palette.hpp"
#include "mamba/core/tasksync.hpp"
#include "mamba/download/mirror_map.hpp"
#include "mamba/download/parameters.hpp"
#include "mamba/fs/filesystem.hpp"
#include "mamba/solver/libsolv/parameters.hpp"
#include "mamba/solver/request.hpp"
@ -77,26 +78,6 @@ namespace mamba
static void use_default_signal_handler(bool val);
struct RemoteFetchParams
{
// ssl_verify can be either an empty string (regular SSL verification),
// the string "<false>" to indicate no SSL verification, or a path to
// a directory with cert files, or a cert file.
std::string ssl_verify{ "" };
bool ssl_no_revoke{ false };
bool curl_initialized{ false }; // non configurable, used in fetch only
std::string user_agent{ "mamba/" LIBMAMBA_VERSION_STRING };
double connect_timeout_secs{ 10. };
// int read_timeout_secs { 60 };
int retry_timeout{ 2 }; // seconds
int retry_backoff{ 3 }; // retry_timeout * retry_backoff
int max_retries{ 3 }; // max number of retries
std::map<std::string, std::string> proxy_servers;
};
struct OutputParams
{
int verbosity{ 0 };
@ -193,7 +174,28 @@ namespace mamba
// micromamba only
bool shell_completion = true;
RemoteFetchParams remote_fetch_params;
download::RemoteFetchParams remote_fetch_params = {
/* .ssl_verify */ { "" },
/* .ssl_no_revoke */ false,
/* .curl_initialized */ false,
/* .user_agent */ { "mamba/" LIBMAMBA_VERSION_STRING },
/* .connect_timeout_secs */ 10.,
/* .retry_timeout */ 2,
/* .retry_backoff */ 3,
/* .max_retries */ 3,
/* .proxy_servers */ {},
};
download::Options download_options() const
{
return {
/* .download_threads */ this->threads_params.download_threads,
/* .fail_fast */ false,
/* .sort */ true,
/* .verbose */ this->output_params.verbosity >= 2,
};
}
OutputParams output_params;
GraphicsParams graphics_params;
SrcParams src_params;

24
libmamba/include/mamba/download/downloader.hpp
View File

@ -7,29 +7,17 @@
#ifndef MAMBA_DOWNLOAD_DOWNLOADER_HPP
#define MAMBA_DOWNLOAD_DOWNLOADER_HPP
#include <functional>
#include <optional>
#include <string>
#include <variant>
#include <tl/expected.hpp>
#include "mamba/core/context.hpp"
#include "mamba/core/error_handling.hpp"
#include "mamba/download/mirror_map.hpp"
#include "mamba/download/parameters.hpp"
#include "mamba/download/request.hpp"
#include "mamba/specs/authentication_info.hpp"
namespace mamba::download
{
struct Options
{
using termination_function = std::optional<std::function<void()>>;
bool fail_fast = false;
bool sort = true;
termination_function on_unexpected_termination = std::nullopt;
};
class Monitor
{
public:
@ -59,7 +47,8 @@ namespace mamba::download
MultiResult download(
MultiRequest requests,
const mirror_map& mirrors,
const Context& context,
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info,
Options options = {},
Monitor* monitor = nullptr
);
@ -67,12 +56,13 @@ namespace mamba::download
Result download(
Request request,
const mirror_map& mirrors,
const Context& context,
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info,
Options options = {},
Monitor* monitor = nullptr
);
bool check_resource_exists(const std::string& url, const Context& context);
bool check_resource_exists(const std::string& url, const RemoteFetchParams& params);
}
#endif

49
libmamba/include/mamba/download/parameters.hpp Normal file
View File

@ -0,0 +1,49 @@
// Copyright (c) 2019, QuantStack and Mamba Contributors
//
// Distributed under the terms of the BSD 3-Clause License.
//
// The full license is in the file LICENSE, distributed with this software.
#ifndef MAMBA_DOWNLOAD_PARAMETERS_HPP
#define MAMBA_DOWNLOAD_PARAMETERS_HPP
#include <functional>
#include <map>
#include <optional>
#include <string>
namespace mamba::download
{
struct RemoteFetchParams
{
// ssl_verify can be either an empty string (regular SSL verification),
// the string "<false>" to indicate no SSL verification, or a path to
// a directory with cert files, or a cert file.
std::string ssl_verify = "";
bool ssl_no_revoke = false;
bool curl_initialized = false; // non configurable, used in fetch only
std::string user_agent = "";
double connect_timeout_secs = 10.;
// int read_timeout_secs { 60 };
int retry_timeout = 2; // seconds
int retry_backoff = 3; // retry_timeout * retry_backoff
int max_retries = 3; // max number of retries
std::map<std::string, std::string> proxy_servers;
};
struct Options
{
using termination_function = std::optional<std::function<void()>>;
std::size_t download_threads = 1;
bool fail_fast = false;
bool sort = true;
bool verbose = false;
termination_function on_unexpected_termination = std::nullopt;
};
}
#endif

8
libmamba/src/api/install.cpp
View File

@ -97,7 +97,13 @@ namespace mamba
url_str,
tmp_file->path()
);
const download::Result res = download::download(std::move(request), ctx.mirrors, ctx);
const download::Result res = download::download(
std::move(request),
ctx.mirrors,
ctx.remote_fetch_params,
ctx.authentication_info(),
ctx.download_options()
);
if (!res || res.value().transfer.http_status != 200)
{

1
libmamba/src/core/package_fetcher.cpp
View File

@ -5,6 +5,7 @@
// The full license is in the file LICENSE, distributed with this software.
#include "mamba/core/invoke.hpp"
#include "mamba/core/output.hpp"
#include "mamba/core/package_fetcher.hpp"
#include "mamba/core/util.hpp"
#include "mamba/specs/archive.hpp"

15
libmamba/src/core/subdirdata.cpp
View File

@ -8,6 +8,7 @@
#include <stdexcept>
#include "mamba/core/channel_context.hpp"
#include "mamba/core/context.hpp"
#include "mamba/core/output.hpp"
#include "mamba/core/package_cache.hpp"
#include "mamba/core/subdirdata.hpp"
@ -517,7 +518,14 @@ namespace mamba
std::move(check_list.begin(), check_list.end(), std::back_inserter(check_requests));
}
}
download::download(std::move(check_requests), context.mirrors, context, {}, check_monitor);
download::download(
std::move(check_requests),
context.mirrors,
context.remote_fetch_params,
context.authentication_info(),
context.download_options(),
check_monitor
);
if (is_sig_interrupted())
{
@ -541,8 +549,9 @@ namespace mamba
download::download(
std::move(index_requests),
context.mirrors,
context,
{ /*fail_fast=*/true },
context.remote_fetch_params,
context.authentication_info(),
context.download_options(),
download_monitor
);
}

12
libmamba/src/core/transaction.cpp
View File

@ -715,7 +715,14 @@ namespace mamba
PackageDownloadMonitor* monitor
)
{
auto result = download::download(std::move(requests), context.mirrors, context, options, monitor);
auto result = download::download(
std::move(requests),
context.mirrors,
context.remote_fetch_params,
context.authentication_info(),
options,
monitor
);
bool all_downloaded = std::all_of(
result.begin(),
result.end(),
@ -779,7 +786,8 @@ namespace mamba
);
std::unique_ptr<PackageDownloadMonitor> monitor = nullptr;
download::Options download_options{ true, true };
auto download_options = ctx.download_options();
download_options.fail_fast = true;
if (PackageDownloadMonitor::can_monitor(ctx))
{
monitor = std::make_unique<PackageDownloadMonitor>();

134
libmamba/src/download/downloader.cpp
View File

@ -39,7 +39,7 @@ namespace mamba::download
"/usr/local/share/certs/ca-root.crt",
};
void init_remote_fetch_params(Context::RemoteFetchParams& remote_fetch_params)
void init_remote_fetch_params(RemoteFetchParams& remote_fetch_params)
{
if (!remote_fetch_params.curl_initialized)
{
@ -145,7 +145,7 @@ namespace mamba::download
bool set_ssl_no_revoke = false;
};
EnvRemoteParams get_env_remote_params(const Context& context)
EnvRemoteParams get_env_remote_params(const RemoteFetchParams& params)
{
// TODO: we should probably store set_low_speed_limit and set_ssl_no_revoke in
// RemoteFetchParams if the request is slower than 30b/s for 60 seconds, cancel.
@ -154,8 +154,7 @@ namespace mamba::download
const bool set_low_speed_opt = (no_low_speed_limit == "0");
const std::string ssl_no_revoke_env = util::get_env("MAMBA_SSL_NO_REVOKE").value_or("0");
const bool set_ssl_no_revoke = context.remote_fetch_params.ssl_no_revoke
|| (ssl_no_revoke_env != "0");
const bool set_ssl_no_revoke = params.ssl_no_revoke || (ssl_no_revoke_env != "0");
return { set_low_speed_opt, set_ssl_no_revoke };
}
@ -169,12 +168,22 @@ namespace mamba::download
CURLHandle& handle,
const MirrorRequest& request,
CURLMultiHandle& downloader,
const Context& context,
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info,
bool verbose,
on_success_callback success,
on_failure_callback error
)
: p_impl(std::make_unique<
Impl>(handle, request, downloader, context, std::move(success), std::move(error)))
: p_impl(std::make_unique<Impl>(
handle,
request,
downloader,
params,
auth_info,
verbose,
std::move(success),
std::move(error)
))
{
}
@ -188,7 +197,9 @@ namespace mamba::download
CURLHandle& handle,
const MirrorRequest& request,
CURLMultiHandle& downloader,
const Context& context,
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info,
bool verbose,
on_success_callback success,
on_failure_callback error
)
@ -196,14 +207,14 @@ namespace mamba::download
, p_request(&request)
, m_success_callback(std::move(success))
, m_failure_callback(std::move(error))
, m_retry_wait_seconds(static_cast<std::size_t>(context.remote_fetch_params.retry_timeout))
, m_retry_wait_seconds(static_cast<std::size_t>(params.retry_timeout))
{
p_stream = make_compression_stream(
p_request->url,
p_request->is_repodata_zst,
[this](char* in, std::size_t size) { return this->write_data(in, size); }
);
configure_handle(context);
configure_handle(params, auth_info, verbose);
downloader.add_handle(*p_handle);
}
@ -313,17 +324,21 @@ namespace mamba::download
}
}
void DownloadAttempt::Impl::configure_handle(const Context& context)
void DownloadAttempt::Impl::configure_handle(
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info,
bool verbose
)
{
const auto [set_low_speed_opt, set_ssl_no_revoke] = get_env_remote_params(context);
const auto [set_low_speed_opt, set_ssl_no_revoke] = get_env_remote_params(params);
p_handle->configure_handle(
util::file_uri_unc2_to_unc4(p_request->url),
set_low_speed_opt,
context.remote_fetch_params.connect_timeout_secs,
params.connect_timeout_secs,
set_ssl_no_revoke,
proxy_match(p_request->url, context.remote_fetch_params.proxy_servers),
context.remote_fetch_params.ssl_verify
proxy_match(p_request->url, params.proxy_servers),
params.ssl_verify
);
if (!p_request->username.empty())
@ -358,24 +373,23 @@ namespace mamba::download
p_handle->add_header("Content-Type: application/json");
}
p_handle->set_opt(CURLOPT_VERBOSE, context.output_params.verbosity >= 2);
p_handle->set_opt(CURLOPT_VERBOSE, verbose);
configure_handle_headers(context);
configure_handle_headers(params, auth_info);
auto logger = spdlog::get("libcurl");
p_handle->set_opt(CURLOPT_DEBUGFUNCTION, curl_debug_callback);
p_handle->set_opt(CURLOPT_DEBUGDATA, logger.get());
}
void DownloadAttempt::Impl::configure_handle_headers(const Context& context)
void DownloadAttempt::Impl::configure_handle_headers(
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info
)
{
p_handle->reset_headers();
std::string user_agent = fmt::format(
"User-Agent: {} {}",
context.remote_fetch_params.user_agent,
curl_version()
);
std::string user_agent = fmt::format("User-Agent: {} {}", params.user_agent, curl_version());
p_handle->add_header(user_agent);
// get url host
@ -389,7 +403,6 @@ namespace mamba::download
// TODO How should this be handled if not empty?
// (think about precedence with request token auth header added below)
const auto& auth_info = context.authentication_info();
if (auto it = auth_info.find_weaken(host); it != auth_info.end())
{
if (const auto& auth = it->second; std::holds_alternative<specs::BearerToken>(auth))
@ -671,7 +684,9 @@ namespace mamba::download
auto MirrorAttempt::prepare_attempt(
CURLHandle& handle,
CURLMultiHandle& downloader,
const Context& context,
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info,
bool verbose,
on_success_callback success,
on_failure_callback error
) -> completion_function
@ -682,7 +697,9 @@ namespace mamba::download
handle,
m_request.value(),
downloader,
context,
params,
auth_info,
verbose,
std::move(success),
std::move(error)
);
@ -798,8 +815,13 @@ namespace mamba::download
}
}
auto DownloadTracker::prepare_new_attempt(CURLMultiHandle& handle, const Context& context)
-> completion_map_entry
auto DownloadTracker::prepare_new_attempt(
CURLMultiHandle& handle,
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info,
bool verbose
) -> completion_map_entry
{
m_state = State::PREPARING;
@ -807,7 +829,9 @@ namespace mamba::download
auto completion_func = m_mirror_attempt.prepare_attempt(
m_handle,
handle,
context,
params,
auth_info,
verbose,
[this](Success res)
{
expected_t<void> finalize_res = invoke_on_success(res);
@ -1031,14 +1055,16 @@ namespace mamba::download
MultiRequest requests,
const mirror_map& mirrors,
Options options,
const Context& context
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info
)
: m_requests(std::move(requests))
, p_mirrors(&mirrors)
, m_options(std::move(options))
, p_context(&context)
, m_curl_handle(context.threads_params.download_threads)
, m_trackers()
, m_curl_handle(options.download_threads)
, m_options(std::move(options))
, p_mirrors(&mirrors)
, p_params(&params)
, p_auth_info(&auth_info)
{
if (m_options.sort)
{
@ -1051,7 +1077,7 @@ namespace mamba::download
}
m_trackers.reserve(m_requests.size());
std::size_t max_retries = static_cast<std::size_t>(context.remote_fetch_params.max_retries);
std::size_t max_retries = static_cast<std::size_t>(params.max_retries);
DownloadTrackerOptions tracker_options{ max_retries, options.fail_fast };
std::transform(
m_requests.begin(),
@ -1089,7 +1115,7 @@ namespace mamba::download
void Downloader::prepare_next_downloads()
{
size_t running_attempts = m_completion_map.size();
const size_t max_parallel_downloads = p_context->threads_params.download_threads;
const size_t max_parallel_downloads = m_options.download_threads;
auto start_filter = mamba::util::filter(
m_trackers,
[&](DownloadTracker& tracker)
@ -1100,7 +1126,7 @@ namespace mamba::download
for (auto& tracker : start_filter)
{
auto [iter, success] = m_completion_map.insert(
tracker.prepare_new_attempt(m_curl_handle, *p_context)
tracker.prepare_new_attempt(m_curl_handle, *p_params, *p_auth_info, m_options.verbose)
);
if (success)
{
@ -1196,30 +1222,31 @@ namespace mamba::download
MultiResult download(
MultiRequest requests,
const mirror_map& mirrors,
const Context& context,
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info,
Options options,
Monitor* monitor
)
{
if (!context.remote_fetch_params.curl_initialized)
if (!params.curl_initialized)
{
// TODO: Move this into an object that would be automatically initialized
// upon construction, and passed by const reference to this function instead
// of context.
Context& ctx = const_cast<Context&>(context);
init_remote_fetch_params(ctx.remote_fetch_params);
auto& params_ = const_cast<RemoteFetchParams&>(params);
init_remote_fetch_params(params_);
}
if (monitor)
{
monitor->observe(requests, options);
on_scope_exit guard([monitor]() { monitor->on_done(); });
Downloader dl(std::move(requests), mirrors, std::move(options), context);
Downloader dl(std::move(requests), mirrors, std::move(options), params, auth_info);
return dl.download();
}
else
{
Downloader dl(std::move(requests), mirrors, std::move(options), context);
Downloader dl(std::move(requests), mirrors, std::move(options), params, auth_info);
return dl.download();
}
}
@ -1227,36 +1254,37 @@ namespace mamba::download
Result download(
Request request,
const mirror_map& mirrors,
const Context& context,
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info,
Options options,
Monitor* monitor
)
{
MultiRequest req(1u, std::move(request));
auto res = download(std::move(req), mirrors, context, std::move(options), monitor);
auto res = download(std::move(req), mirrors, params, auth_info, std::move(options), monitor);
return std::move(res.front());
}
bool check_resource_exists(const std::string& url, const Context& context)
bool check_resource_exists(const std::string& url, const RemoteFetchParams& params)
{
if (!context.remote_fetch_params.curl_initialized)
if (!params.curl_initialized)
{
// TODO: Move this into an object that would be automatically initialized
// upon construction, and passed by const reference to this function instead
// of context.
Context& ctx = const_cast<Context&>(context);
init_remote_fetch_params(ctx.remote_fetch_params);
auto& params_ = const_cast<RemoteFetchParams&>(params);
init_remote_fetch_params(params_);
}
const auto [set_low_speed_opt, set_ssl_no_revoke] = get_env_remote_params(context);
const auto [set_low_speed_opt, set_ssl_no_revoke] = get_env_remote_params(params);
return curl::check_resource_exists(
util::file_uri_unc2_to_unc4(url),
set_low_speed_opt,
context.remote_fetch_params.connect_timeout_secs,
params.connect_timeout_secs,
set_ssl_no_revoke,
proxy_match(url, context.remote_fetch_params.proxy_servers),
context.remote_fetch_params.ssl_verify
proxy_match(url, params.proxy_servers),
params.ssl_verify
);
}
}

48
libmamba/src/download/downloader_impl.hpp
View File

@ -8,11 +8,14 @@
#define MAMBA_DL_DOWNLOADER_IMPL_HPP
#include <chrono>
#include <fstream>
#include <optional>
#include <unordered_map>
#include "mamba/download/downloader.hpp"
#include "mamba/download/mirror_map.hpp"
#include "mamba/download/parameters.hpp"
#include "mamba/specs/authentication_info.hpp"
#include "mamba/util/flat_set.hpp"
#include "compression.hpp"
@ -37,7 +40,9 @@ namespace mamba::download
CURLHandle& handle,
const MirrorRequest& request,
CURLMultiHandle& downloader,
const Context& context,
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info,
bool verbose,
on_success_callback success,
on_failure_callback error
);
@ -55,7 +60,9 @@ namespace mamba::download
CURLHandle& handle,
const MirrorRequest& request,
CURLMultiHandle& downloader,
const Context& context,
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info,
bool verbose,
on_success_callback success,
on_failure_callback error
);
@ -64,8 +71,15 @@ namespace mamba::download
void clean_attempt(CURLMultiHandle& downloader, bool erase_downloaded);
void invoke_progress_callback(const Event&) const;
void configure_handle(const Context& context);
void configure_handle_headers(const Context& context);
void configure_handle(
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info,
bool verbose
);
void configure_handle_headers(
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info
);
size_t write_data(char* buffer, size_t data);
@ -128,7 +142,9 @@ namespace mamba::download
auto prepare_attempt(
CURLHandle& handle,
CURLMultiHandle& downloader,
const Context& context,
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info,
bool verbose,
on_success_callback success,
on_failure_callback error
) -> completion_function;
@ -193,8 +209,12 @@ namespace mamba::download
DownloadTrackerOptions options
);
auto prepare_new_attempt(CURLMultiHandle& handle, const Context& context)
-> completion_map_entry;
auto prepare_new_attempt(
CURLMultiHandle& handle,
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info,
bool verbose
) -> completion_map_entry;
bool has_failed() const;
bool can_start_transfer() const;
@ -261,7 +281,8 @@ namespace mamba::download
MultiRequest requests,
const mirror_map& mirrors,
Options options,
const Context& context
const RemoteFetchParams& params,
const specs::AuthenticationDataBase& auth_info
);
MultiResult download();
@ -275,12 +296,13 @@ namespace mamba::download
void invoke_unexpected_termination() const;
MultiRequest m_requests;
const mirror_map* p_mirrors;
Options m_options;
const Context* p_context;
CURLMultiHandle m_curl_handle;
std::vector<DownloadTracker> m_trackers;
size_t m_waiting_count;
CURLMultiHandle m_curl_handle;
Options m_options;
const mirror_map* p_mirrors;
const RemoteFetchParams* p_params;
const specs::AuthenticationDataBase* p_auth_info;
std::size_t m_waiting_count;
using completion_function = DownloadTracker::completion_function;
std::unordered_map<CURLId, completion_function> m_completion_map;

7
libmamba/src/validation/repo_checker.cpp
View File

@ -6,6 +6,7 @@
#include <string>
#include "mamba/core/context.hpp"
#include "mamba/core/output.hpp"
#include "mamba/core/timeref.hpp"
#include "mamba/core/util.hpp"
@ -224,14 +225,16 @@ namespace mamba::validation
auto url = util::concat(m_base_url, "/", f.string());
tmp_file_path = tmp_dir_path / f;
if (download::check_resource_exists(url, m_context))
if (download::check_resource_exists(url, m_context.get().remote_fetch_params))
{
download::Request
request(f.string(), download::MirrorName(""), url, tmp_file_path.string());
download::Result res = download::download(
std::move(request),
m_context.get().mirrors,
m_context
m_context.get().remote_fetch_params,
m_context.get().authentication_info(),
m_context.get().download_options()
);
if (res)

26
libmamba/src/validation/update_framework_v0_6.cpp
View File

@ -7,6 +7,7 @@
#include <fstream>
#include <utility>
#include "mamba/core/context.hpp"
#include "mamba/core/output.hpp"
#include "mamba/core/util.hpp"
#include "mamba/download/downloader.hpp"
@ -204,7 +205,7 @@ namespace mamba::validation::v0_6
.value()
/ "key_mgr.json";
if (download::check_resource_exists(url.pretty_str(), context))
if (download::check_resource_exists(url.pretty_str(), context.remote_fetch_params))
{
download::Request request(
"key_mgr.json",
@ -212,7 +213,18 @@ namespace mamba::validation::v0_6
url.str(util::URL::Credentials::Show),
tmp_metadata_path.string()
);
download::Result res = download::download(std::move(request), context.mirrors, context);
download::Result res = download::download(
std::move(request),
context.mirrors,
context.remote_fetch_params,
context.authentication_info(),
download::Options{
/* .download_threads */ context.threads_params.download_threads,
/* .fail_fast */ false,
/* .sort */ true,
/* .verbose */ context.output_params.verbosity >= 2,
}
);
if (res)
{
KeyMgrRole key_mgr = create_key_mgr(tmp_metadata_path);
@ -365,7 +377,7 @@ namespace mamba::validation::v0_6
const auto url = mamba::util::URL::parse(base_url + "/pkg_mgr.json").value();
if (download::check_resource_exists(url.pretty_str(), context))
if (download::check_resource_exists(url.pretty_str(), context.remote_fetch_params))
{
download::Request request(
"pkg_mgr.json",
@ -373,7 +385,13 @@ namespace mamba::validation::v0_6
url.pretty_str(),
tmp_metadata_path.string()
);
download::Result res = download::download(std::move(request), context.mirrors, context);
download::Result res = download::download(
std::move(request),
context.mirrors,
context.remote_fetch_params,
context.authentication_info(),
context.download_options()
);
if (res)
{

47
libmamba/tests/src/download/test_downloader.cpp
View File

@ -7,79 +7,68 @@
#include <catch2/catch_all.hpp>
#include "mamba/api/configuration.hpp"
#include "mamba/core/util.hpp"
#include "mamba/download/downloader.hpp"
#include "mambatests.hpp"
#include "mamba/util/string.hpp"
namespace mamba
{
namespace
{
TEST_CASE("file_does_not_exist")
TEST_CASE("file_does_not_exist", "[mamba::download]")
{
download::Request request(
"test",
download::MirrorName(""),
"file:///nonexistent/repodata.json",
"test_download_repodata.json",
"test_download_repodata_1.json",
false,
true
);
auto& context = mambatests::singletons().context;
const auto previous_quiet = context.output_params.quiet;
auto _ = on_scope_exit([&] { context.output_params.quiet = previous_quiet; });
download::MultiRequest dl_request{ std::vector{ std::move(request) } };
context.output_params.quiet = true;
download::MultiResult res = download::download(dl_request, context.mirrors, context);
download::MultiResult res = download::download(dl_request, {}, {}, {});
REQUIRE(res.size() == std::size_t(1));
REQUIRE(!res[0]);
REQUIRE(res[0].error().attempt_number == std::size_t(1));
}
TEST_CASE("file_does_not_exist_throw")
TEST_CASE("file_does_not_exist_throw", "[mamba::download]")
{
download::Request request(
"test",
download::MirrorName(""),
"file:///nonexistent/repodata.json",
"test_download_repodata.json"
"test_download_repodata_2.json"
);
download::MultiRequest dl_request{ std::vector{ std::move(request) } };
auto& context = mambatests::singletons().context;
const auto previous_quiet = context.output_params.quiet;
auto _ = on_scope_exit([&] { context.output_params.quiet = previous_quiet; });
context.output_params.quiet = true;
REQUIRE_THROWS_AS(
download::download(dl_request, context.mirrors, context),
std::runtime_error
);
REQUIRE_THROWS_AS(download::download(dl_request, {}, {}, {}), std::runtime_error);
}
TEST_CASE("Use CA certificate from the root prefix")
TEST_CASE("Use CA certificate from the root prefix", "[mamba::download]")
{
// Create a context, make a request and check that ssl_verify is set to the correct path
auto& context = mambatests::singletons().context;
const auto tmp_dir = TemporaryDirectory();
// Set the context values to the default ones
context.remote_fetch_params.curl_initialized = false;
context.remote_fetch_params.ssl_verify = "<system>";
auto params = download::RemoteFetchParams{};
params.curl_initialized = false;
params.ssl_verify = "<system>";
download::Request request(
"test",
download::MirrorName(""),
"https://conda.anaconda.org/conda-forge/linux-64/repodata.json",
"test_download_repodata.json"
tmp_dir.path() / "test_download_repodata_3.json"
);
download::MultiRequest dl_request{ std::vector{ std::move(request) } };
// Downloading must initialize curl and set `ssl_verify` to the path of the CA
// certificate
REQUIRE(!context.remote_fetch_params.curl_initialized);
download::MultiResult res = download::download(dl_request, context.mirrors, context);
REQUIRE(context.remote_fetch_params.curl_initialized);
REQUIRE(!params.curl_initialized);
download::MultiResult res = download::download(dl_request, {}, params, {});
REQUIRE(params.curl_initialized);
auto certificates = context.remote_fetch_params.ssl_verify;
auto certificates = params.ssl_verify;
const fs::u8path root_prefix = detail::get_root_prefix();
const fs::u8path expected_certificates = root_prefix / "ssl" / "cert.pem";

18
libmamba/tests/src/download/test_mirror.cpp
View File

@ -18,35 +18,35 @@ namespace mamba::download
namespace
{
TEST_CASE("tar_bz2_extension")
TEST_CASE("tar_bz2_extension", "[mamba::download]")
{
auto [split_path, split_tag] = split_path_tag("xtensor-0.23.10-h2acdbc0_0.tar.bz2");
REQUIRE(split_path == "xtensor");
REQUIRE(split_tag == "0.23.10-h2acdbc0-0");
}
TEST_CASE("multiple_parts")
TEST_CASE("multiple_parts", "[mamba::download]")
{
auto [split_path, split_tag] = split_path_tag("x-tensor-10.23.10-h2acdbc0_0.tar.bz2");
REQUIRE(split_path == "x-tensor");
REQUIRE(split_tag == "10.23.10-h2acdbc0-0");
}
TEST_CASE("more_multiple_parts")
TEST_CASE("more_multiple_parts", "[mamba::download]")
{
auto [split_path, split_tag] = split_path_tag("x-tens-or-10.23.10-h2acdbc0_0.tar.bz2");
REQUIRE(split_path == "x-tens-or");
REQUIRE(split_tag == "10.23.10-h2acdbc0-0");
}
TEST_CASE("json_extension")
TEST_CASE("json_extension", "[mamba::download]")
{
auto [split_path, split_tag] = split_path_tag("xtensor-0.23.10-h2acdbc0_0.json");
REQUIRE(split_path == "xtensor-0.23.10-h2acdbc0_0.json");
REQUIRE(split_tag == "latest");
}
TEST_CASE("not_enough_parts")
TEST_CASE("not_enough_parts", "[mamba::download]")
{
REQUIRE_THROWS_AS(split_path_tag("xtensor.tar.bz2"), std::runtime_error);
}
@ -55,7 +55,7 @@ namespace mamba::download
namespace
{
TEST_CASE("PassThroughMirror")
TEST_CASE("PassThroughMirror", "[mamba::download]")
{
std::unique_ptr<Mirror> mir = make_mirror("");
// `mir_ref` is used here to provide an explicit expression to `typeid`
@ -73,7 +73,7 @@ namespace mamba::download
REQUIRE(mir_req.url == "linux-64/repodata.json");
}
TEST_CASE("HTTPMirror")
TEST_CASE("HTTPMirror", "[mamba::download]")
{
SECTION("https")
{
@ -142,7 +142,7 @@ namespace mamba::download
}
}
TEST_CASE("OCIMirror")
TEST_CASE("OCIMirror", "[mamba::download]")
{
SECTION("Request repodata.json")
{
@ -208,7 +208,7 @@ namespace mamba::download
}
}
TEST_CASE("nullptr")
TEST_CASE("nullptr", "[mamba::download]")
{
std::unique_ptr<Mirror> mir = make_mirror("ghcr.io/channel-mirrors/conda-forge");
REQUIRE(mir == nullptr);

16
libmambapy/src/libmambapy/bindings/legacy.cpp
View File

@ -741,16 +741,16 @@ bind_submodule_impl(pybind11::module_ m)
.def("set_verbosity", &Context::set_verbosity)
.def("set_log_level", &Context::set_log_level);
py::class_<Context::RemoteFetchParams>(ctx, "RemoteFetchParams")
py::class_<download::RemoteFetchParams>(ctx, "RemoteFetchParams")
.def(py::init<>())
.def_readwrite("ssl_verify", &Context::RemoteFetchParams::ssl_verify)
.def_readwrite("max_retries", &Context::RemoteFetchParams::max_retries)
.def_readwrite("retry_timeout", &Context::RemoteFetchParams::retry_timeout)
.def_readwrite("retry_backoff", &Context::RemoteFetchParams::retry_backoff)
.def_readwrite("user_agent", &Context::RemoteFetchParams::user_agent)
.def_readwrite("ssl_verify", &download::RemoteFetchParams::ssl_verify)
.def_readwrite("max_retries", &download::RemoteFetchParams::max_retries)
.def_readwrite("retry_timeout", &download::RemoteFetchParams::retry_timeout)
.def_readwrite("retry_backoff", &download::RemoteFetchParams::retry_backoff)
.def_readwrite("user_agent", &download::RemoteFetchParams::user_agent)
// .def_readwrite("read_timeout_secs", &Context::RemoteFetchParams::read_timeout_secs)
.def_readwrite("proxy_servers", &Context::RemoteFetchParams::proxy_servers)
.def_readwrite("connect_timeout_secs", &Context::RemoteFetchParams::connect_timeout_secs);
.def_readwrite("proxy_servers", &download::RemoteFetchParams::proxy_servers)
.def_readwrite("connect_timeout_secs", &download::RemoteFetchParams::connect_timeout_secs);
py::class_<Context::OutputParams>(ctx, "OutputParams")
.def(py::init<>())