folkslinux
/
mamba
mirror of https://github.com/mamba-org/mamba.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve ChannelContext and Channel (#3003)

This commit is contained in:
Antoine Prouvost 2023-11-23 19:14:35 +01:00 committed by GitHub
parent acd357df95
commit 1340d712eb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 669 additions and 752 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
libmamba/include/mamba/core/channel_context.hpp
View File

@ -22,28 +22,48 @@ namespace mamba
public:
using ChannelResolveParams = specs::ChannelResolveParams;
using Channel = specs::Channel;
using channel_list = ChannelResolveParams::channel_list;
/**
* Create a ChannelContext with a simple parsing of the context options.
*
* No hardcoded names are added.
* Custom channels are treated as aliases rather than the Conda way (the name is not
* added at the end of the URL if absent).
*/
[[nodiscard]] static auto make_simple(Context& ctx) -> ChannelContext;
/**
* Create a ChannelContext while applying all of Conda context options.
*
* If not defined, the Conda custom channels "pkgs/main", "pkgs/r", "pkgs/pro",
* and "pkgs/msys2" (Windows only) will be added.
* If not defined, the Conda custom mutlit channels "defaults" and "local" will
* be added.
* The function will ensure custom channels names are added at the end of the URLs.
*/
[[nodiscard]] static auto make_conda_compatible(Context& ctx) -> ChannelContext;
/**
* Initialize channel with the paramters as they are.
* Initialize channel with the parameters as they are.
*
* The Context is not parsed.
*/
ChannelContext(Context& ctx, ChannelResolveParams params);
ChannelContext(Context& ctx, ChannelResolveParams params, std::vector<Channel> has_zst);
auto make_channel(std::string_view name) -> const channel_list&;
[[nodiscard]] auto params() const -> const specs::ChannelResolveParams&;
[[nodiscard]] auto context() const -> const Context&;
[[nodiscard]] auto has_zst(const Channel& chan) const -> bool;
/**
* Return the context.
*
* @deprecated We aim to remove the capture of the Context.
*/
[[nodiscard, deprecated]] auto context() const -> const Context&;
private:
@ -51,6 +71,7 @@ namespace mamba
ChannelResolveParams m_channel_params;
ChannelCache m_channel_cache;
std::vector<Channel> m_has_zst;
std::reference_wrapper<const Context> m_context;
};
}

14
libmamba/include/mamba/specs/channel.hpp
View File

@ -8,7 +8,6 @@
#include <string>
#include <string_view>
#include <unordered_map>
#include <utility>
#include <vector>
#include "mamba/specs/authentication_info.hpp"
@ -90,10 +89,16 @@ namespace mamba::specs
Channel(CondaURL url, std::string display_name, util::flat_set<std::string> platforms = {});
[[nodiscard]] auto is_package() const -> bool;
[[nodiscard]] auto url() const -> const CondaURL&;
auto clear_url() -> const CondaURL;
void set_url(CondaURL url);
[[nodiscard]] auto platform_urls() const -> std::vector<CondaURL>;
[[nodiscard]] auto platform_url(std::string_view platform) const -> CondaURL;
[[nodiscard]] auto platforms() const -> const platform_list&;
auto clear_platforms() -> platform_list;
void set_platforms(platform_list platforms);
@ -108,13 +113,6 @@ namespace mamba::specs
[[nodiscard]] auto contains_equivalent(const Channel& other) const -> bool;
[[nodiscard]] auto
platform_url(std::string_view platform, bool with_credential = true) const -> std::string;
// The pairs consist of (platform,url)
[[nodiscard]] auto platform_urls(bool with_credential = true) const
-> util::flat_set<std::pair<std::string, std::string>>;
[[nodiscard]] auto urls(bool with_credential = true) const -> util::flat_set<std::string>;
private:
CondaURL m_url;

4
libmamba/src/api/channel_loader.cpp
View File

@ -65,13 +65,13 @@ namespace mamba
{
for (auto channel : pool.channel_context().make_channel(location))
{
for (auto& [platform, url] : channel.platform_urls(true))
for (const auto& platform : channel.platforms())
{
auto sdires = MSubdirData::create(
pool.channel_context(),
channel,
platform,
url,
channel.platform_url(platform).str(),
package_caches,
"repodata.json"
);

51
libmamba/src/api/configuration.cpp
View File

@ -884,43 +884,6 @@ namespace mamba
return paths;
}
void custom_channels_hook(std::map<std::string, std::string>& custom_channels)
{
// Hard coded Anaconda channels names.
// This will not redefine them if the user has already defined these keys.
custom_channels.emplace("pkgs/main", "https://repo.anaconda.com/pkgs/main");
custom_channels.emplace("pkgs/r", "https://repo.anaconda.com/pkgs/r");
custom_channels.emplace("pkgs/pro", "https://repo.anaconda.com/pkgs/pro");
if (util::on_win)
{
custom_channels.emplace("pkgs/msys2", "https://repo.anaconda.com/pkgs/msys2");
}
}
void custom_multichannels_hook(
const Context& context,
std::map<std::string, std::vector<std::string>>& custom_multichannels
)
{
custom_multichannels.emplace("defaults", context.default_channels);
auto local_channels = std::vector<std::string>();
local_channels.reserve(3);
for (auto p : {
context.prefix_params.target_prefix / "conda-bld",
context.prefix_params.root_prefix / "conda-bld",
fs::u8path(util::user_home_dir()) / "conda-bld",
})
{
if (fs::exists(p))
{
local_channels.push_back(std::move(p));
}
}
custom_multichannels.emplace("local", std::move(local_channels));
}
void pkgs_dirs_hook(std::vector<fs::u8path>& dirs)
{
for (auto& d : dirs)
@ -1315,11 +1278,7 @@ namespace mamba
.description("Custom channels")
.long_description( //
"A dictionary with name: url to use for custom channels.\n"
"If not defined, the Conda special names "
R"("pkgs/main", "pkgs/r", "pkgs/pro", and "pkgs/msys2" (Windows only) )"
"will be added."
)
.set_post_merge_hook(detail::custom_channels_hook));
));
insert(Configurable("custom_multichannels", &m_context.custom_multichannels)
.group("Channels")
@ -1328,14 +1287,8 @@ namespace mamba
.long_description( //
"A dictionary where keys are multi channels names, and values are a list "
"of correspinding names / urls / file paths to use.\n"
R"(If not defined, the Conda special mutli channels "defaults" is added )"
R"(with values from the "default_channels" option, and "local" is added )"
R"(with "~/conda-bld" and target and root prefix "conda-bld" subfolders/)"
)
.needs({ "default_channels", "target_prefix", "root_prefix" })
.set_post_merge_hook<std::map<std::string, std::vector<std::string>>>(
[this](auto& val) { detail::custom_multichannels_hook(m_context, val); }
));
.needs({ "default_channels", "target_prefix", "root_prefix" }));
insert(Configurable("override_channels_enabled", &m_context.override_channels_enabled)
.group("Channels")

4
libmamba/src/api/info.cpp
View File

@ -167,9 +167,9 @@ namespace mamba
{
for (auto channel : channel_context.make_channel(loc))
{
for (auto url : channel.urls(true))
for (auto url : channel.platform_urls())
{
channel_urls.push_back(url);
channel_urls.push_back(std::move(url).str());
}
}
}

121
libmamba/src/core/channel_context.cpp
View File

@ -4,6 +4,8 @@
//
// The full license is in the file LICENSE, distributed with this software.
#include <algorithm>
#include <array>
#include <cassert>
#include <tuple>
#include <utility>
@ -63,6 +65,24 @@ namespace mamba
}
}
template <bool on_win>
auto conda_custom_channels()
{
using namespace std::literals::string_view_literals;
static constexpr std::size_t count = 3 + (on_win ? 1 : 0);
auto channels = std::array<std::pair<std::string_view, std::string_view>, count>{
std::pair{ "pkgs/main"sv, "https://repo.anaconda.com/pkgs/main"sv },
std::pair{ "pkgs/r"sv, "https://repo.anaconda.com/pkgs/r"sv },
std::pair{ "pkgs/pro"sv, "https://repo.anaconda.com/pkgs/pro"sv },
};
if constexpr (on_win)
{
channels[3] = std::pair{ "pkgs/msys2"sv, "https://repo.anaconda.com/pkgs/msys2"sv };
}
return channels;
}
void add_conda_params_custom_channel(specs::ChannelResolveParams& params, const Context& ctx)
{
for (const auto& [name, location] : ctx.custom_channels)
@ -78,6 +98,15 @@ namespace mamba
chan.set_display_name(name);
params.custom_channels.emplace(name, std::move(chan));
}
// Hard coded Anaconda channels names.
// This will not redefine them if the user has already defined these keys.
for (const auto& [name, location] : conda_custom_channels<util::on_win>())
{
auto chan = make_unique_chan(location, params);
chan.set_display_name(std::string(name));
params.custom_channels.emplace(name, std::move(chan));
}
}
void
@ -94,11 +123,77 @@ namespace mamba
params.custom_multichannels.emplace(multi_name, std::move(channels));
}
}
void
add_conda_params_custom_multichannel(specs::ChannelResolveParams& params, const Context& ctx)
{
// Hard coded Anaconda "defaults" multi channel name.
// This will not redefine them if the user has already defined these keys.
if (auto it = ctx.custom_multichannels.find("defaults");
it == ctx.custom_multichannels.cend())
{
auto channels = specs::ChannelResolveParams::channel_list();
channels.reserve(ctx.default_channels.size());
std::transform(
ctx.default_channels.cbegin(),
ctx.default_channels.cend(),
std::back_inserter(channels),
[&params](const auto& loc) { return make_unique_chan(loc, params); }
);
params.custom_multichannels.emplace("defaults", std::move(channels));
}
// Hard coded Anaconda "local" multi channel name.
// This will not redefine them if the user has already defined these keys.
if (auto it = ctx.custom_multichannels.find("local");
it == ctx.custom_multichannels.cend())
{
auto channels = specs::ChannelResolveParams::channel_list();
channels.reserve(3);
for (auto path : {
ctx.prefix_params.target_prefix / "conda-bld",
ctx.prefix_params.root_prefix / "conda-bld",
fs::u8path(params.home_dir) / "conda-bld",
})
{
if (fs::exists(path))
{
channels.push_back(make_unique_chan(path.string(), params));
}
}
params.custom_multichannels.emplace("local", std::move(channels));
}
// Called after to guarentee there are no custom multichannels when calling
// make_unique_chan.
add_simple_params_custom_multichannel(params, ctx);
}
auto create_zstd(const Context& ctx, specs::ChannelResolveParams params)
-> std::vector<specs::Channel>
{
auto out = std::vector<specs::Channel>();
if (ctx.repodata_use_zst)
{
out.reserve(ctx.repodata_has_zst.size());
for (const auto& loc : ctx.repodata_has_zst)
{
auto spec = specs::ChannelSpec::parse(loc);
auto channels = specs::Channel::resolve(std::move(spec), params);
for (auto& chan : channels)
{
out.push_back(std::move(chan));
}
}
}
return out;
}
}
ChannelContext::ChannelContext(Context& context, ChannelResolveParams params)
ChannelContext::ChannelContext(Context& ctx, ChannelResolveParams params, std::vector<Channel> has_zst)
: m_channel_params(std::move(params))
, m_context(context)
, m_has_zst(has_zst)
, m_context(ctx)
{
}
@ -107,15 +202,17 @@ namespace mamba
auto params = make_simple_params_base(ctx);
add_simple_params_custom_channel(params, ctx);
add_simple_params_custom_multichannel(params, ctx);
return { ctx, std::move(params) };
auto has_zst = create_zstd(ctx, params);
return { ctx, std::move(params), std::move(has_zst) };
}
auto ChannelContext::make_conda_compatible(Context& ctx) -> ChannelContext
{
auto params = make_simple_params_base(ctx);
add_conda_params_custom_channel(params, ctx);
add_simple_params_custom_multichannel(params, ctx);
return { ctx, std::move(params) };
add_conda_params_custom_multichannel(params, ctx);
auto has_zst = create_zstd(ctx, params);
return { ctx, std::move(params), has_zst };
}
auto ChannelContext::make_channel(std::string_view name) -> const channel_list&
@ -127,7 +224,7 @@ namespace mamba
auto [it, inserted] = m_channel_cache.emplace(
name,
specs::Channel::resolve(specs::ChannelSpec::parse(name), params())
Channel::resolve(specs::ChannelSpec::parse(name), params())
);
assert(inserted);
return it->second;
@ -138,6 +235,18 @@ namespace mamba
return m_channel_params;
}
auto ChannelContext::has_zst(const Channel& chan) const -> bool
{
for (const auto& zst_chan : m_has_zst)
{
if (zst_chan.contains_equivalent(chan))
{
return true;
}
}
return false;
}
auto ChannelContext::context() const -> const Context&
{
return m_context;

2
libmamba/src/core/package_fetcher.cpp
View File

@ -80,7 +80,7 @@ namespace mamba
{
auto channels = channel_context.make_channel(pkg_info.url);
assert(channels.size() == 1); // A URL can only resolve to one channel
m_url = channels.front().urls(true)[0];
m_url = channels.front().platform_urls().at(0).str();
}
else
{

2
libmamba/src/core/prefix_data.cpp
View File

@ -169,7 +169,7 @@ namespace mamba
// If someone wrote multichannel names in repodata_record, we don't know which one is the
// correct URL. This is must never happen!
assert(channels.size() == 1);
prec.channel = channels.front().platform_url(prec.subdir);
prec.channel = channels.front().platform_url(prec.subdir).str();
m_package_records.insert({ prec.name, std::move(prec) });
}
} // namespace mamba

27
libmamba/src/core/subdirdata.cpp
View File

@ -378,23 +378,6 @@ namespace mamba
return max_age;
}
bool check_zst(ChannelContext& channel_context, const specs::Channel& channel)
{
// TODO the list of channels with zst should really be computed only once in
// the ChannelContext
for (const auto& c : channel_context.context().repodata_has_zst)
{
for (const auto& chan : channel_context.make_channel(c))
{
if (chan.contains_equivalent(channel))
{
return true;
}
}
}
return false;
}
fs::u8path get_cache_dir(const fs::u8path& cache_path)
{
return cache_path / "cache";
@ -676,15 +659,7 @@ namespace mamba
const Context& context = channel_context.context();
if (!context.offline || forbid_cache(m_repodata_url))
{
if (context.repodata_use_zst)
{
bool has_zst = m_metadata.has_zst();
if (!has_zst)
{
has_zst = check_zst(channel_context, channel);
m_metadata.set_zst(has_zst);
}
}
m_metadata.set_zst(m_metadata.has_zst() || channel_context.has_zst(channel));
}
}

98
libmamba/src/specs/channel.cpp
View File

@ -46,6 +46,11 @@ namespace mamba::specs
m_url.set_path(std::move(p));
}
auto Channel::is_package() const -> bool
{
return !url().package().empty();
}
auto Channel::url() const -> const CondaURL&
{
return m_url;
@ -61,6 +66,32 @@ namespace mamba::specs
m_url = std::move(url);
}
auto Channel::platform_urls() const -> std::vector<CondaURL>
{
if (is_package())
{
return { url() };
}
auto out = std::vector<CondaURL>();
out.reserve(platforms().size());
for (const auto& platform : platforms())
{
out.push_back(platform_url(platform));
}
return out;
}
auto Channel::platform_url(std::string_view platform) const -> CondaURL
{
if (is_package())
{
return url();
}
return (url() / platform);
}
auto Channel::platforms() const -> const platform_list&
{
return m_platforms;
@ -115,50 +146,6 @@ namespace mamba::specs
return url_equivalent_with(other) && util::set_is_superset_of(platforms(), other.platforms());
}
auto Channel::urls(bool with_credential) const -> util::flat_set<std::string>
{
if (!url().package().empty())
{
return { url().str(
with_credential ? CondaURL::Credentials::Show : CondaURL::Credentials::Remove
) };
}
auto out = util::flat_set<std::string>{};
for (auto& [_, v] : platform_urls(with_credential))
{
out.insert(v);
}
return out;
}
auto Channel::platform_urls(bool with_credential) const
-> util::flat_set<std::pair<std::string, std::string>>
{
if (!url().package().empty())
{
return {};
}
auto out = util::flat_set<std::pair<std::string, std::string>>{};
for (const auto& platform : platforms())
{
out.insert({ platform, platform_url(platform, with_credential) });
}
return out;
}
auto Channel::platform_url(std::string_view platform, bool with_credential) const -> std::string
{
auto cred = with_credential ? CondaURL::Credentials::Show : CondaURL::Credentials::Remove;
if (!url().package().empty())
{
return url().str(cred);
}
return (url() / platform).str(cred);
}
/****************************************
* Implementation of Channel::resolve *
****************************************/
@ -324,22 +311,13 @@ namespace mamba::specs
auto resolve_name_in_custom_channel(
ChannelSpec&& spec,
ChannelResolveParamsView params,
const Channel& match
std::string_view match_name,
const Channel& match_chan
) -> Channel
{
auto url = match.url();
// we can have a channel like
// testchannel: https://server.com/private/testchannel
// where `name == private/testchannel` and we need to join the remaining label part
// of the channel (e.g. -c testchannel/mylabel/xyz)
// needs to result in https://server.com/private/testchannel/mylabel/xyz
std::string combined_name = util::concat_dedup_splits(
util::rstrip(url.path(), '/'),
util::lstrip(spec.location(), '/'),
'/'
);
url.set_path(combined_name);
auto url = match_chan.url();
url.append_path(util::remove_prefix(spec.location(), match_name));
set_fallback_credential_from_db(url, params.authentication_db);
return {
/* url= */ std::move(url),
@ -387,7 +365,9 @@ namespace mamba::specs
if (auto it = params.custom_channels.find_weaken(spec.location());
it != params.custom_channels.cend())
{
return { resolve_name_in_custom_channel(std::move(spec), params, it->second) };
return {
resolve_name_in_custom_channel(std::move(spec), params, it->first, it->second)
};
}
if (auto it = params.custom_multichannels.find(spec.location());

977
libmamba/tests/src/core/test_channel_context.cpp
View File

File diff suppressed because it is too large Load Diff

12
libmamba/tests/src/core/test_satisfiability_error.cpp
View File

@ -334,11 +334,15 @@ namespace
{
for (const auto& chan : pool.channel_context().make_channel(location))
{
for (auto& [platform, url] : chan.platform_urls(true))
for (const auto& platform : chan.platforms())
{
auto sub_dir = expected_value_or_throw(
MSubdirData::create(pool.channel_context(), chan, platform, url, cache)
);
auto sub_dir = expected_value_or_throw(MSubdirData::create(
pool.channel_context(),
chan,
platform,
chan.platform_url(platform).str(),
cache
));
sub_dirs.push_back(std::move(sub_dir));
}
}

82
libmamba/tests/src/specs/test_channel.cpp
View File

@ -22,9 +22,9 @@ TEST_SUITE("specs::channel")
using platform_list = Channel::platform_list;
using namespace std::literals::string_view_literals;
TEST_CASE("Channel constructor")
TEST_CASE("Channel")
{
SUBCASE("Trailing slash")
SUBCASE("Constructor railing slash")
{
// Leading slash for empty paths
for (auto url : {
@ -32,6 +32,7 @@ TEST_SUITE("specs::channel")
"https://repo.mamba.pm"sv,
})
{
CAPTURE(url);
auto chan = Channel(CondaURL::parse(url), "somename");
CHECK_NE(chan.url().str(), mamba::util::rstrip(url, '/'));
}
@ -43,10 +44,83 @@ TEST_SUITE("specs::channel")
"ftp://mamba.org/some/folder"sv,
})
{
CAPTURE(url);
auto chan = Channel(CondaURL::parse(url), "somename");
CHECK_EQ(chan.url().str(), mamba::util::rstrip(url, '/'));
}
}
SUBCASE("Equality")
{
for (auto raw_url : {
"https://repo.mamba.pm/"sv,
"https://repo.mamba.pm"sv,
"https://repo.mamba.pm/conda-forge/win-64/"sv,
"file:///some/folder/"sv,
"ftp://mamba.org/some/folder"sv,
})
{
CAPTURE(raw_url);
auto chan_a = Channel(CondaURL::parse(raw_url), "somename", { "linux-64" });
CHECK_EQ(chan_a, chan_a);
auto chan_b = chan_a;
CHECK_EQ(chan_b, chan_a);
CHECK_EQ(chan_a, chan_b);
chan_b = chan_a;
chan_b.set_platforms({ "linux-64", "noarch" });
CHECK_NE(chan_b, chan_a);
chan_b = chan_a;
chan_b.set_display_name("othername");
CHECK_NE(chan_b, chan_a);
}
}
SUBCASE("Equivalence")
{
for (auto raw_url : {
"https://repo.mamba.pm/"sv,
"https://repo.mamba.pm/t/mytoken/"sv,
"https://user:pass@repo.mamba.pm/conda-forge/win-64/"sv,
"file:///some/folder/"sv,
"ftp://mamba.org/some/folder"sv,
})
{
CAPTURE(raw_url);
auto url_a = CondaURL::parse(raw_url);
auto url_b = url_a;
url_b.clear_user();
url_b.clear_password();
url_b.clear_token();
auto chan_a = Channel(url_a, "somename", { "linux-64" });
auto chan_b = Channel(url_b, "somename", { "linux-64" });
// Channel::url_equivalent_with
CHECK(chan_a.url_equivalent_with(chan_a));
CHECK(chan_b.url_equivalent_with(chan_b));
CHECK(chan_a.url_equivalent_with(chan_b));
CHECK(chan_b.url_equivalent_with(chan_a));
// Channel::contains_equivalent
CHECK(chan_a.contains_equivalent(chan_a));
CHECK(chan_b.contains_equivalent(chan_b));
CHECK(chan_a.contains_equivalent(chan_b));
CHECK(chan_b.contains_equivalent(chan_a));
chan_a.set_platforms({ "noarch", "linux-64" });
CHECK(chan_a.contains_equivalent(chan_a));
CHECK(chan_a.contains_equivalent(chan_b));
CHECK_FALSE(chan_b.contains_equivalent(chan_a));
chan_b.set_platforms({ "osx-64" });
CHECK_FALSE(chan_a.contains_equivalent(chan_b));
CHECK_FALSE(chan_b.contains_equivalent(chan_a));
}
}
}
TEST_CASE("Channel::resolve")
@ -525,8 +599,8 @@ TEST_SUITE("specs::channel")
auto channels = Channel::resolve(specs, params);
REQUIRE_EQ(channels.size(), 1);
const auto& chan = channels.front();
// Higher precedence. Unfotunate, but the name must be repeated...
CHECK_EQ(chan.url(), CondaURL::parse("ftp://mydomain.net/conda/conda-forge"));
// Higher precedence.
CHECK_EQ(chan.url(), CondaURL::parse("ftp://mydomain.net/conda"));
CHECK_EQ(chan.display_name(), name);
CHECK_EQ(chan.platforms(), params.platforms);
}