folkslinux
/
mamba
mirror of https://github.com/mamba-org/mamba.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Solver improvements (#3140) 

* Change serializing of pin

* Change pin names

* Rename MSolverProblem > SolverProblem

* Move pool_add_matchspec to helpers

* Move transaction_to_solution to helpers

* Remove MSolver::add_constraints

* Move pool_add_pin to helpers

* Remove MSolver pointer convertion

* Remove MSolver macro flags
This commit is contained in:
Antoine Prouvost 2024-01-19 10:04:45 +01:00 committed by GitHub
parent 81504b5614
commit e8793e59e6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 514 additions and 469 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
libmamba/include/mamba/core/error_handling.hpp
View File

@ -32,6 +32,7 @@ namespace mamba
satisfiablitity_error,
user_interrupted,
incorrect_usage,
invalid_spec
};
class mamba_error : public std::runtime_error

18
libmamba/include/mamba/core/solver.hpp
View File

@ -24,15 +24,6 @@
#include "mamba/specs/match_spec.hpp"
#include "mamba/specs/package_info.hpp"
#define PY_MAMBA_NO_DEPS 0b0001
#define PY_MAMBA_ONLY_DEPS 0b0010
#define PY_MAMBA_FORCE_REINSTALL 0b0100
extern "C"
{
typedef struct s_Solver Solver;
}
namespace mamba::solv
{
class ObjQueue;
@ -42,7 +33,7 @@ namespace mamba::solv
namespace mamba
{
struct MSolverProblem
struct SolverProblem
{
SolverRuleinfo type;
Id source_id;
@ -78,12 +69,9 @@ namespace mamba
void add_global_job(int job_flag);
void add_jobs(const std::vector<std::string>& jobs, int job_flag);
void add_constraint(const std::string& job);
void add_pin(const std::string& pin);
void add_pins(const std::vector<std::string>& pins);
[[deprecated]] void py_set_postsolve_flags(const std::vector<std::pair<int, int>>& flags);
void set_flags(const Flags& flags); // TODO temporary Itf meant to be passed in ctor
[[nodiscard]] auto flags() const -> const Flags&;
[[deprecated]] void py_set_libsolv_flags(const std::vector<std::pair<int, int>>& flags);
@ -94,7 +82,7 @@ namespace mamba
[[nodiscard]] std::string problems_to_str() const;
[[nodiscard]] std::vector<std::string> all_problems() const;
[[nodiscard]] std::vector<MSolverProblem> all_problems_structured() const;
[[nodiscard]] std::vector<SolverProblem> all_problems_structured() const;
[[nodiscard]] ProblemsGraph problems_graph() const;
[[nodiscard]] std::string all_problems_to_str() const;
std::ostream& explain_problems(std::ostream& out) const;
@ -109,8 +97,6 @@ namespace mamba
[[nodiscard]] const std::vector<specs::MatchSpec>& neuter_specs() const;
[[nodiscard]] const std::vector<specs::MatchSpec>& pinned_specs() const;
operator const Solver*() const;
operator Solver*();
auto solver() -> solv::ObjSolver&;
auto solver() const -> const solv::ObjSolver&;

145
libmamba/src/core/pool.cpp
View File

@ -148,148 +148,11 @@ namespace mamba
return solvables.as<std::vector>();
}
namespace
auto MPool::matchspec2id(const specs::MatchSpec& ms) -> ::Id
{
auto
channel_match(const std::vector<specs::Channel>& ms_channels, const specs::CondaURL& pkg_url)
-> specs::Channel::Match
{
auto match = specs::Channel::Match::No;
// More than one element means the channel spec was a custom_multi_channel
for (const auto& chan : ms_channels)
{
switch (chan.contains_package(pkg_url))
{
case specs::Channel::Match::Full:
return specs::Channel::Match::Full;
case specs::Channel::Match::InOtherPlatform:
// Keep looking for full matches
match = specs::Channel::Match::InOtherPlatform;
break;
case specs::Channel::Match::No:
// No overriding potential InOtherPlatform match
break;
}
}
return match;
}
/**
* Add function to handle matchspec while parsing is done by libsolv.
*/
auto add_channel_specific_matchspec(
ChannelContext& channel_context,
solv::ObjPool& pool,
const specs::MatchSpec& ms
) -> solv::DependencyId
{
assert(ms.channel().has_value());
// Poor man's ms repr to match waht the user provided
const std::string repr = fmt::format("{}::{}", *ms.channel(), ms.conda_build_form());
// Already added, return that id
if (const auto maybe_id = pool.find_string(repr))
{
return maybe_id.value();
}
// conda_build_form does **NOT** contain the channel info
const solv::DependencyId match = pool_conda_matchspec(
pool.raw(),
ms.conda_build_form().c_str()
);
auto ms_channels = channel_context.make_channel(*ms.channel());
solv::ObjQueue selected_pkgs = {};
auto other_subdir_match = std::string();
pool.for_each_whatprovides(
match,
[&](solv::ObjSolvableViewConst s)
{
if (s.installed())
{
// This will have the effect that channel-specific MatchSpec will always be
// reinstalled.
// This is not the intended behaviour but an historical artifact on which
// ``--force-reinstall`` currently rely.
return;
}
assert(ms.channel().has_value());
const auto match = channel_match(ms_channels, specs::CondaURL::parse(s.url()));
switch (match)
{
case (specs::Channel::Match::Full):
{
selected_pkgs.push_back(s.id());
break;
}
case (specs::Channel::Match::InOtherPlatform):
{
other_subdir_match = s.subdir();
break;
}
case (specs::Channel::Match::No):
{
break;
}
}
}
);
if (selected_pkgs.empty())
{
if (!other_subdir_match.empty())
{
const auto& filters = ms.channel()->platform_filters();
throw std::runtime_error(fmt::format(
R"(The package "{}" is not available for the specified platform{} ({}))"
R"( but is available on {}.)",
ms.str(),
filters.size() > 1 ? "s" : "",
fmt::join(filters, ", "),
other_subdir_match
));
}
else
{
throw std::runtime_error(fmt::format(
R"(The package "{}" is not found in any loaded channels.)"
R"( Try adding more channels or subdirs.)",
ms.str()
));
}
}
const solv::StringId repr_id = pool.add_string(repr);
// FRAGILE This get deleted when calling ``pool_createwhatprovides`` so care
// must be taken to do it before
// TODO investigate namespace providers
pool.add_to_whatprovides(repr_id, pool.add_to_whatprovides_data(selected_pkgs));
return repr_id;
}
}
::Id MPool::matchspec2id(const specs::MatchSpec& ms)
{
::Id id = 0;
if (!ms.channel().has_value())
{
id = pool_conda_matchspec(pool().raw(), ms.conda_build_form().c_str());
}
else
{
// Working around shortcomings of ``pool_conda_matchspec``
// The channels are not processed.
// TODO Fragile! Installing this matchspec will always trigger a reinstall
id = add_channel_specific_matchspec(channel_context(), pool(), ms);
}
if (id == 0)
{
throw std::runtime_error("libsolv error: could not create matchspec from string");
}
return id;
return solver::libsolv::pool_add_matchspec(pool(), ms, channel_context().params())
.or_else([](mamba_error&& error) { throw std::move(error); })
.value_or(0);
}
std::optional<specs::PackageInfo> MPool::id2pkginfo(Id solv_id) const

131
libmamba/src/core/solver.cpp
View File

@ -5,7 +5,6 @@
// The full license is in the file LICENSE, distributed with this software.
#include <sstream>
#include <stdexcept>
#include <fmt/format.h>
#include <fmt/ostream.h>
@ -26,6 +25,8 @@
#include "solv-cpp/queue.hpp"
#include "solv-cpp/solver.hpp"
#include "solver/libsolv/helpers.hpp"
namespace mamba
{
MSolver::MSolver(MPool pool, std::vector<std::pair<int, int>> flags)
@ -45,16 +46,6 @@ namespace mamba
MSolver& MSolver::operator=(MSolver&&) = default;
MSolver::operator const Solver*() const
{
return solver().raw();
}
MSolver::operator Solver*()
{
return solver().raw();
}
auto MSolver::solver() -> solv::ObjSolver&
{
return *m_solver;
@ -167,81 +158,24 @@ namespace mamba
}
}
void MSolver::add_constraint(const std::string& job)
{
m_jobs->push_back(
SOLVER_INSTALL | SOLVER_SOLVABLE_PROVIDES,
m_pool.matchspec2id(specs::MatchSpec::parse(job))
);
}
void MSolver::add_pin(const std::string& pin)
{
// In libsolv, locking means that a package keeps the same state: if it is installed,
// it remains installed, if not it remains uninstalled.
// Locking on a spec applies the lock to all packages matching the spec.
// In mamba, we do not want to lock the package because we want to allow other variants
// (matching the same spec) to unlock more solutions.
// For instance we may pin ``libfmt=8.*`` but allow it to be swaped with a version built
// by a more recent compiler.
//
// A previous version of this function would use ``SOLVER_LOCK`` to lock all packages not
// matching the pin.
// That played poorly with ``all_problems_structured`` because we could not interpret
// the ids that were returned (since they were not associated with a single reldep).
//
// Another wrong idea is to add the pin as an install job.
// This is not what is expected of pins, as they must not be installed if they were not
// in the environement.
// They can be configure in ``.condarc`` for generally specifying what versions are wanted.
//
// The idea behind the current version is to add the pin/spec as a constraint that must be
// fullfield only if the package is installed.
// This is not supported on solver jobs but it is on ``Solvable`` with
// ``disttype == DISTYPE_CONDA``.
// Therefore, we add a dummy solvable marked as already installed, and add the pin/spec
// as one of its constrains.
// Then we lock this solvable and force the re-checking of its dependencies.
const auto pin_ms = specs::MatchSpec::parse(pin);
m_pinned_specs.push_back(pin_ms);
solver::libsolv::pool_add_pin(m_pool.pool(), pin_ms, m_pool.channel_context().params())
.transform(
[&](solv::ObjSolvableView pin_solv)
{
m_pinned_specs.push_back(std::move(pin_ms));
auto& pool = m_pool.pool();
if (pool.disttype() != DISTTYPE_CONDA)
{
throw std::runtime_error("Cannot add pin to a pool that is not of Conda distype");
}
auto installed = pool.installed_repo();
if (!installed.has_value())
{
throw std::runtime_error("Cannot add pin without a repo of installed packages");
}
// Add dummy solvable with a constraint on the pin (not installed if not present)
auto [cons_solv_id, cons_solv] = installed->add_solvable();
// TODO set some "pin" key on the solvable so that we can retrieve it during error messages
const std::string cons_solv_name = fmt::format("pin-{}", m_pinned_specs.size());
cons_solv.set_name(cons_solv_name);
cons_solv.set_version("1");
cons_solv.add_constraints(solv::ObjQueue{ m_pool.matchspec2id(pin_ms) });
// Solvable need to provide itself
cons_solv.add_self_provide();
// Even if we lock it, libsolv may still try to remove it with
// `SOLVER_FLAG_ALLOW_UNINSTALL`, so we flag it as not a real package to filter it out in
// the transaction
cons_solv.set_artificial(true);
// Necessary for attributes to be properly stored
installed->internalize();
// WARNING keep separate or libsolv does not understand
// Force verify the dummy solvable dependencies, as this is not the default for
// installed packages.
add_jobs({ cons_solv_name }, SOLVER_VERIFY);
// Lock the dummy solvable so that it stays install.
add_jobs({ cons_solv_name }, SOLVER_LOCK);
// WARNING keep separate or libsolv does not understand
// Force verify the dummy solvable dependencies, as this is not the default for
// installed packages.
add_jobs({ std::string(pin_solv.name()) }, SOLVER_VERIFY);
// Lock the dummy solvable so that it stays install.
add_jobs({ std::string(pin_solv.name()) }, SOLVER_LOCK);
}
)
.or_else([](mamba_error&& error) { throw std::move(error); });
}
void MSolver::add_pins(const std::vector<std::string>& pins)
@ -252,25 +186,6 @@ namespace mamba
}
}
void MSolver::py_set_postsolve_flags(const std::vector<std::pair<int, int>>& flags)
{
for (const auto& option : flags)
{
switch (option.first)
{
case PY_MAMBA_NO_DEPS:
m_flags.keep_dependencies = !option.second;
break;
case PY_MAMBA_ONLY_DEPS:
m_flags.keep_specs = !option.second;
break;
case PY_MAMBA_FORCE_REINSTALL:
m_flags.force_reinstall = option.second;
break;
}
}
}
void MSolver::set_flags(const Flags& flags)
{
m_flags = flags;
@ -291,7 +206,7 @@ namespace mamba
// TODO use new API
for (const auto& option : m_libsolv_flags)
{
solver_set_flag(*this, option.first, option.second);
solver_set_flag(m_solver->raw(), option.first, option.second);
}
}
@ -363,7 +278,7 @@ namespace mamba
namespace
{
// TODO change MSolver problem
MSolverProblem make_solver_problem(
SolverProblem make_solver_problem(
const MSolver& solver,
const MPool& pool,
SolverRuleinfo type,
@ -372,7 +287,6 @@ namespace mamba
Id dep_id
)
{
const ::Solver* const solver_ptr = solver;
return {
/* .type= */ type,
/* .source_id= */ source_id,
@ -383,7 +297,8 @@ namespace mamba
/* .dep= */ pool.dep2str(dep_id),
/* .description= */
solver_problemruleinfo2str(
const_cast<::Solver*>(solver_ptr), // Not const because might alloctmp space
const_cast<::Solver*>(solver.solver().raw()), // Not const because might
// alloctmp space
type,
source_id,
target_id,
@ -393,9 +308,9 @@ namespace mamba
}
}
std::vector<MSolverProblem> MSolver::all_problems_structured() const
std::vector<SolverProblem> MSolver::all_problems_structured() const
{
std::vector<MSolverProblem> res = {};
std::vector<SolverProblem> res = {};
res.reserve(solver().problem_count()); // Lower bound
solver().for_each_problem_id(
[&](solv::ProblemId pb)
@ -479,7 +394,7 @@ namespace mamba
namespace
{
void warn_unexpected_problem(const MSolverProblem& problem)
void warn_unexpected_problem(const SolverProblem& problem)
{
// TODO: Once the new error message are not experimental, we should consider
// reducing this level since it is not somethig the user has control over.

156
libmamba/src/core/transaction.cpp
View File

@ -16,11 +16,6 @@
#include <fmt/ostream.h>
#include <solv/selection.h>
extern "C" // Incomplete header
{
#include <solv/conda.h>
}
#include "mamba/core/channel_context.hpp"
#include "mamba/core/context.hpp"
#include "mamba/core/download_progress_bar.hpp"
@ -42,6 +37,8 @@ extern "C" // Incomplete header
#include "solv-cpp/solver.hpp"
#include "solv-cpp/transaction.hpp"
#include "solver/libsolv/helpers.hpp"
#include "progress_bar_impl.hpp"
namespace mamba
@ -95,7 +92,7 @@ namespace mamba
auto specs_names(const MSolver& solver) -> util::flat_set<std::string>
{
// TODO C++20
// TODO(C++20):
// to_install_names and to_remove_names need not be allocated, only that
// flat_set::insert with iterators is more efficient (because it sorts only once).
// This could be solved with std::range::transform
@ -127,134 +124,6 @@ namespace mamba
return specs;
}
auto transaction_to_solution(
const MPool& pool,
const solv::ObjTransaction& trans,
const util::flat_set<std::string>& specs = {},
/** true to filter out specs, false to filter in specs */
bool keep_only = true
) -> Solution
{
auto get_pkginfo = [&](solv::SolvableId id)
{
const auto pkginfo = pool.id2pkginfo(id);
assert(pkginfo.has_value());
return std::move(pkginfo).value();
};
auto get_newer_pkginfo = [&](solv::SolvableId id)
{
auto maybe_newer_id = trans.step_newer(pool.pool(), id);
assert(maybe_newer_id.has_value());
return get_pkginfo(maybe_newer_id.value());
};
auto out = Solution::action_list();
out.reserve(trans.size());
trans.for_each_step_id(
[&](const solv::SolvableId id)
{
auto pkginfo = get_pkginfo(id);
// In libsolv, system dependencies are provided as a special dependency,
// while in Conda it is implemented as a virtual package.
// Maybe there is a way to tell libsolv to never try to install or remove these
// solvables (SOLVER_LOCK or SOLVER_USERINSTALLED?).
// In the meantime (and probably later for safety) we filter all virtual
// packages out.
if (util::starts_with(pkginfo.name, "__")) // i.e. is_virtual_package
{
return;
}
// Artificial packages are packages that were added to implement a feature
// (e.g. a pin) but do not represent a Conda package.
// They can appear in the transaction depending on libsolv flags.
// We use this attribute to filter them out.
if (const auto solv = pool.pool().get_solvable(id);
solv.has_value() && solv->artificial())
{
LOG_DEBUG << "Solution: Remove artificial " << pkginfo.str();
return;
}
// keep_only ? specs.contains(...) : !specs.contains(...);
// TODO ideally we should use Matchspecs::contains(pkginfo)
if (keep_only == specs.contains(pkginfo.name))
{
LOG_DEBUG << "Solution: Omit " << pkginfo.str();
out.push_back(Solution::Omit{ std::move(pkginfo) });
return;
}
auto const type = trans.step_type(
pool.pool(),
id,
SOLVER_TRANSACTION_SHOW_OBSOLETES | SOLVER_TRANSACTION_OBSOLETE_IS_UPGRADE
);
switch (type)
{
case SOLVER_TRANSACTION_UPGRADED:
{
auto newer = get_newer_pkginfo(id);
LOG_DEBUG << "Solution: Upgrade " << pkginfo.str() << " -> "
<< newer.str();
out.push_back(Solution::Upgrade{
/* .remove= */ std::move(pkginfo),
/* .install= */ std::move(newer),
});
break;
}
case SOLVER_TRANSACTION_CHANGED:
{
auto newer = get_newer_pkginfo(id);
LOG_DEBUG << "Solution: Change " << pkginfo.str() << " -> "
<< newer.str();
out.push_back(Solution::Change{
/* .remove= */ std::move(pkginfo),
/* .install= */ std::move(newer),
});
break;
}
case SOLVER_TRANSACTION_REINSTALLED:
{
LOG_DEBUG << "Solution: Reinstall " << pkginfo.str();
out.push_back(Solution::Reinstall{ std::move(pkginfo) });
break;
}
case SOLVER_TRANSACTION_DOWNGRADED:
{
auto newer = get_newer_pkginfo(id);
LOG_DEBUG << "Solution: Downgrade " << pkginfo.str() << " -> "
<< newer.str();
out.push_back(Solution::Downgrade{
/* .remove= */ std::move(pkginfo),
/* .install= */ std::move(newer),
});
break;
}
case SOLVER_TRANSACTION_ERASE:
{
LOG_DEBUG << "Solution: Remove " << pkginfo.str();
out.push_back(Solution::Remove{ std::move(pkginfo) });
break;
}
case SOLVER_TRANSACTION_INSTALL:
{
LOG_DEBUG << "Solution: Install " << pkginfo.str();
out.push_back(Solution::Install{ std::move(pkginfo) });
break;
}
case SOLVER_TRANSACTION_IGNORE:
break;
default:
LOG_WARNING << "solv::ObjTransaction case not handled: " << type;
break;
}
}
);
return { std::move(out) };
}
auto find_python_version(const Solution& solution, const solv::ObjPool& pool)
-> std::pair<std::string, std::string>
{
@ -366,7 +235,7 @@ namespace mamba
// TODO reload dependency information from ``ctx.target_prefix / "conda-meta"`` after
// ``fetch_extract_packages`` is called.
m_solution = transaction_to_solution(m_pool, trans);
m_solution = solver::libsolv::transaction_to_solution(m_pool.pool(), trans);
m_history_entry.remove.reserve(specs_to_remove.size());
for (auto& s : specs_to_remove)
@ -414,11 +283,16 @@ namespace mamba
const auto& flags = solver.flags();
if (flags.keep_specs && flags.keep_dependencies)
{
m_solution = transaction_to_solution(m_pool, trans);
m_solution = solver::libsolv::transaction_to_solution(m_pool.pool(), trans);
}
else
{
m_solution = transaction_to_solution(m_pool, trans, specs_names(solver), !(flags.keep_specs));
m_solution = solver::libsolv::transaction_to_solution(
m_pool.pool(),
trans,
specs_names(solver),
!(flags.keep_specs)
);
}
if (solver.flags().keep_specs)
@ -538,12 +412,12 @@ namespace mamba
// Init solution again...
if (flags.keep_specs && flags.keep_dependencies)
{
m_solution = transaction_to_solution(m_pool, trans);
m_solution = solver::libsolv::transaction_to_solution(m_pool.pool(), trans);
}
else
{
m_solution = transaction_to_solution(
m_pool,
m_solution = solver::libsolv::transaction_to_solution(
m_pool.pool(),
trans,
specs_names(solver),
!(flags.keep_specs)
@ -591,7 +465,7 @@ namespace mamba
auto trans = solv::ObjTransaction::from_solvables(m_pool.pool(), decision);
trans.order(m_pool.pool());
m_solution = transaction_to_solution(m_pool, trans);
m_solution = solver::libsolv::transaction_to_solution(m_pool.pool(), trans);
std::vector<specs::MatchSpec> specs_to_install;
for (const auto& pkginfo : packages)

8
libmamba/src/solv-cpp/pool.cpp
View File

@ -128,13 +128,7 @@ namespace mamba::solv
auto ObjPool::add_conda_dependency(raw_str_view dep) -> DependencyId
{
if (const auto id = ::pool_conda_matchspec(raw(), dep); id != 0)
{
return id;
}
auto msg = std::stringstream{};
msg << R"(Invalid conda dependency: ")" << dep << '"';
throw std::invalid_argument(msg.str());
return ::pool_conda_matchspec(raw(), dep);
}
auto ObjPool::add_conda_dependency(const std::string& dep) -> DependencyId

29
libmamba/src/solv-cpp/solvable.cpp
View File

@ -466,28 +466,21 @@ namespace mamba::solv
return (repo != nullptr) && (repo == repo->pool->installed);
}
namespace
auto ObjSolvableViewConst::type() const -> SolvableType
{
auto solvable_lookup_bool(const ::Solvable* s, ::Id key) -> bool
{
return ::solvable_lookup_num(const_cast<::Solvable*>(s), key, 0) != 0;
}
void solvable_set_bool(::Solvable* s, ::Id key, bool val)
{
::solvable_set_num(s, key, (val ? 1 : 0));
}
using Num = std::underlying_type_t<SolvableType>;
// (Ab)using meaningless key
return static_cast<SolvableType>(::solvable_lookup_num(
const_cast<::Solvable*>(raw()),
SOLVABLE_INSTALLSTATUS,
static_cast<Num>(SolvableType::Package)
));
}
auto ObjSolvableViewConst::artificial() const -> bool
void ObjSolvableView::set_type(SolvableType val) const
{
using Num = std::underlying_type_t<SolvableType>;
// (Ab)using meaningless key
return solvable_lookup_bool(raw(), SOLVABLE_INSTALLSTATUS);
}
void ObjSolvableView::set_artificial(bool val) const
{
// (Ab)using meaningless key
::solvable_set_num(raw(), SOLVABLE_INSTALLSTATUS, val);
::solvable_set_num(raw(), SOLVABLE_INSTALLSTATUS, static_cast<Num>(val));
}
}

26
libmamba/src/solv-cpp/solvable.hpp
View File

@ -9,7 +9,6 @@
#include <string>
#include <string_view>
#include <utility>
#include <solv/pooltypes.h>
@ -23,6 +22,16 @@ extern "C"
namespace mamba::solv
{
/**
* We use solvable for all sort of things, including virtual packages and pins.
*/
enum class SolvableType : unsigned long long
{
Package,
Virtualpackage,
Pin,
};
class ObjSolvableViewConst
{
public:
@ -93,13 +102,8 @@ namespace mamba::solv
/** Whether the solvable is in the installed repo. */
auto installed() const -> bool;
/**
* Some artificial packages are added to produce extra features (e.g. pins).
*
* We flag them as such so that we avoid trying to install them.
* This as no effect on libsolv, it must be checked manually.
*/
auto artificial() const -> bool;
/** The type for which the solvable is used. */
auto type() const -> SolvableType;
private:
@ -403,11 +407,11 @@ namespace mamba::solv
void add_track_features(const Range& features) const;
/**
* Mark package as artificial, i.e. that must not be installed.
* Mark mark the package as being of a specific type.
*
* @see ObjSolvableViewConst::artificial
* @see ObjSolvableViewConst::type
*/
void set_artificial(bool val) const;
void set_type(SolvableType val) const;
};
/***************************************

372
libmamba/src/solver/libsolv/helpers.cpp
View File

@ -19,6 +19,7 @@
#include "mamba/core/util.hpp"
#include "mamba/specs/conda_url.hpp"
#include "mamba/util/cfile.hpp"
#include "mamba/util/random.hpp"
#include "mamba/util/string.hpp"
#include "solver/libsolv/helpers.hpp"
@ -114,6 +115,14 @@ namespace mamba::solver::libsolv
std::transform(feats.begin(), feats.end(), std::back_inserter(out.track_features), id_to_str);
}
// Pins have a name like "pin-fsej43208fsd" so we set a readable name for them.
// This is mainly displayed in the solver error messages.
// Perhaps this is not the best place to put this...
if (s.type() == solv::SolvableType::Pin)
{
out.name = fmt::format("pin on {}", fmt::join(out.constrains, " and "));
}
return out;
}
@ -390,12 +399,10 @@ namespace mamba::solver::libsolv
solv::ObjPool& pool,
solv::ObjRepoView repo,
const fs::u8path& filename,
const solver::libsolv::RepodataOrigin& expected,
const RepodataOrigin& expected,
bool expected_pip_added
) -> expected_t<solv::ObjRepoView>
{
using RepodataOrigin = solver::libsolv::RepodataOrigin;
static constexpr auto expected_binary_version = std::string_view(MAMBA_SOLV_VERSION);
LOG_INFO << "Attempting to read libsolv solv file " << filename << " for repo "
@ -492,8 +499,7 @@ namespace mamba::solver::libsolv
);
}
auto
write_solv(solv::ObjRepoView repo, fs::u8path filename, const solver::libsolv::RepodataOrigin& metadata)
auto write_solv(solv::ObjRepoView repo, fs::u8path filename, const RepodataOrigin& metadata)
-> expected_t<solv::ObjRepoView>
{
LOG_INFO << "Writing libsolv solv file " << filename << " for repo " << repo.name();
@ -565,4 +571,360 @@ namespace mamba::solver::libsolv
);
repo.set_pip_added(true);
}
namespace
{
auto
channel_match(const std::vector<specs::Channel>& ms_channels, const specs::CondaURL& pkg_url)
-> specs::Channel::Match
{
auto match = specs::Channel::Match::No;
// More than one element means the channel spec was a custom_multi_channel
for (const auto& chan : ms_channels)
{
switch (chan.contains_package(pkg_url))
{
case specs::Channel::Match::Full:
return specs::Channel::Match::Full;
case specs::Channel::Match::InOtherPlatform:
// Keep looking for full matches
match = specs::Channel::Match::InOtherPlatform;
break;
case specs::Channel::Match::No:
// No overriding potential InOtherPlatform match
break;
}
}
return match;
}
/**
* Add function to handle matchspec while parsing is done by libsolv.
*/
auto add_channel_specific_matchspec(
solv::ObjPool& pool,
const specs::MatchSpec& ms,
const specs::ChannelResolveParams& params
) -> solv::DependencyId
{
assert(ms.channel().has_value());
const std::string repr = ms.str();
// Already added, return that id
if (const auto maybe_id = pool.find_string(repr))
{
return maybe_id.value();
}
// conda_build_form does **NOT** contain the channel info
const solv::DependencyId match = pool_conda_matchspec(
pool.raw(),
ms.conda_build_form().c_str()
);
auto ms_channels = specs::Channel::resolve(*ms.channel(), params);
solv::ObjQueue selected_pkgs = {};
auto other_subdir_match = std::string();
pool.for_each_whatprovides(
match,
[&](solv::ObjSolvableViewConst s)
{
if (s.installed())
{
// This will have the effect that channel-specific MatchSpec will always be
// reinstalled.
// This is not the intended behaviour but an historical artifact on which
// ``--force-reinstall`` currently rely.
return;
}
assert(ms.channel().has_value());
const auto match = channel_match(ms_channels, specs::CondaURL::parse(s.url()));
switch (match)
{
case (specs::Channel::Match::Full):
{
selected_pkgs.push_back(s.id());
break;
}
case (specs::Channel::Match::InOtherPlatform):
{
other_subdir_match = s.subdir();
break;
}
case (specs::Channel::Match::No):
{
break;
}
}
}
);
if (selected_pkgs.empty())
{
if (!other_subdir_match.empty())
{
const auto& filters = ms.channel()->platform_filters();
throw std::runtime_error(fmt::format(
R"(The package "{}" is not available for the specified platform{} ({}))"
R"( but is available on {}.)",
ms.str(),
filters.size() > 1 ? "s" : "",
fmt::join(filters, ", "),
other_subdir_match
));
}
else
{
throw std::runtime_error(fmt::format(
R"(The package "{}" is not found in any loaded channels.)"
R"( Try adding more channels or subdirs.)",
ms.str()
));
}
}
const solv::StringId repr_id = pool.add_string(repr);
// FRAGILE This get deleted when calling ``pool_createwhatprovides`` so care
// must be taken to do it before
// TODO investigate namespace providers
pool.add_to_whatprovides(repr_id, pool.add_to_whatprovides_data(selected_pkgs));
return repr_id;
}
}
[[nodiscard]] auto pool_add_matchspec( //
solv::ObjPool& pool,
const specs::MatchSpec& ms,
const specs::ChannelResolveParams& params
) -> expected_t<solv::DependencyId>
{
solv::DependencyId id = 0;
if (!ms.channel().has_value())
{
id = pool.add_conda_dependency(ms.conda_build_form());
}
else
{
// Working around shortcomings of ``pool_conda_matchspec``
// The channels are not processed.
// TODO Fragile! Installing this matchspec will always trigger a reinstall
id = add_channel_specific_matchspec(pool, ms, params);
}
if (id == 0)
{
make_unexpected(
fmt::format(R"(Invalid MatchSpec "{}")", ms.str()),
mamba_error_code::invalid_spec
);
}
return id;
}
auto pool_add_pin( //
solv::ObjPool& pool,
const specs::MatchSpec& pin,
const specs::ChannelResolveParams& params
) -> expected_t<solv::ObjSolvableView>
{
// In libsolv, locking means that a package keeps the same state: if it is installed,
// it remains installed, if not it remains uninstalled.
// Locking on a spec applies the lock to all packages matching the spec.
// In mamba, we do not want to lock the package because we want to allow other variants
// (matching the same spec) to unlock more solutions.
// For instance we may pin ``libfmt=8.*`` but allow it to be swaped with a version built
// by a more recent compiler.
//
// A previous version of this function would use ``SOLVER_LOCK`` to lock all packages not
// matching the pin.
// That played poorly with ``all_problems_structured`` because we could not interpret
// the ids that were returned (since they were not associated with a single reldep).
//
// Another wrong idea is to add the pin as an install job.
// This is not what is expected of pins, as they must not be installed if they were not
// in the environement.
// They can be configure in ``.condarc`` for generally specifying what versions are wanted.
//
// The idea behind the current version is to add the pin/spec as a constraint that must be
// fullfield only if the package is installed.
// This is not supported on solver jobs but it is on ``Solvable`` with
// ``disttype == DISTYPE_CONDA``.
// Therefore, we add a dummy solvable marked as already installed, and add the pin/spec
// as one of its constrains.
// Then we lock this solvable and force the re-checking of its dependencies.
if (pool.disttype() != DISTTYPE_CONDA)
{
return make_unexpected(
fmt::format(R"(Cannot add pin "{}" to a pool that is not of Conda distype)", pin.str()),
mamba_error_code::incorrect_usage
);
}
auto installed = pool.installed_repo();
if (!installed.has_value())
{
return make_unexpected(
fmt::format(R"("Cannot add pin "{}" without a repo of installed packages")", pin.str()),
mamba_error_code::incorrect_usage
);
}
return pool_add_matchspec(pool, pin, params)
.transform(
[&](solv::DependencyId cons)
{
// Add dummy solvable with a constraint on the pin (not installed if not
// present)
auto [cons_solv_id, cons_solv] = installed->add_solvable();
const std::string cons_solv_name = fmt::format(
"pin-{}",
util::generate_random_alphanumeric_string(10)
);
cons_solv.set_name(cons_solv_name);
cons_solv.set_version("1");
cons_solv.add_constraint(cons);
// Solvable need to provide itself
cons_solv.add_self_provide();
// Even if we lock it, libsolv may still try to remove it with
// `SOLVER_FLAG_ALLOW_UNINSTALL`, so we flag it as not a real package to filter
// it out in the transaction
cons_solv.set_type(solv::SolvableType::Pin);
// Necessary for attributes to be properly stored
// TODO move this at the end of all job requests
installed->internalize();
return cons_solv;
}
);
}
auto transaction_to_solution(
const solv::ObjPool& pool,
const solv::ObjTransaction& trans,
const util::flat_set<std::string>& specs,
/** true to filter out specs, false to filter in specs */
bool keep_only
) -> Solution
{
auto get_pkginfo = [&](solv::SolvableId id)
{
assert(pool.get_solvable(id).has_value());
return make_package_info(pool, pool.get_solvable(id).value());
};
auto get_newer_pkginfo = [&](solv::SolvableId id)
{
auto maybe_newer_id = trans.step_newer(pool, id);
assert(maybe_newer_id.has_value());
return get_pkginfo(maybe_newer_id.value());
};
auto out = Solution::action_list();
out.reserve(trans.size());
trans.for_each_step_id(
[&](const solv::SolvableId id)
{
auto pkginfo = get_pkginfo(id);
// In libsolv, system dependencies are provided as a special dependency,
// while in Conda it is implemented as a virtual package.
// Maybe there is a way to tell libsolv to never try to install or remove these
// solvables (SOLVER_LOCK or SOLVER_USERINSTALLED?).
// In the meantime (and probably later for safety) we filter all virtual
// packages out.
if (util::starts_with(pkginfo.name, "__")) // i.e. is_virtual_package
{
return;
}
// here are packages that were added to implement a feature
// (e.g. a pin) but do not represent a Conda package.
// They can appear in the transaction depending on libsolv flags.
// We use this attribute to filter them out.
if (const auto solv = pool.get_solvable(id);
solv.has_value() && (solv->type() != solv::SolvableType::Package))
{
LOG_DEBUG << "Solution: Remove artificial " << pkginfo.str();
return;
}
// keep_only ? specs.contains(...) : !specs.contains(...);
// TODO ideally we should use Matchspecs::contains(pkginfo)
if (keep_only == specs.contains(pkginfo.name))
{
LOG_DEBUG << "Solution: Omit " << pkginfo.str();
out.emplace_back(Solution::Omit{ std::move(pkginfo) });
return;
}
auto const type = trans.step_type(
pool,
id,
SOLVER_TRANSACTION_SHOW_OBSOLETES | SOLVER_TRANSACTION_OBSOLETE_IS_UPGRADE
);
switch (type)
{
case SOLVER_TRANSACTION_UPGRADED:
{
auto newer = get_newer_pkginfo(id);
LOG_DEBUG << "Solution: Upgrade " << pkginfo.str() << " -> " << newer.str();
out.emplace_back(Solution::Upgrade{
/* .remove= */ std::move(pkginfo),
/* .install= */ std::move(newer),
});
break;
}
case SOLVER_TRANSACTION_CHANGED:
{
auto newer = get_newer_pkginfo(id);
LOG_DEBUG << "Solution: Change " << pkginfo.str() << " -> " << newer.str();
out.emplace_back(Solution::Change{
/* .remove= */ std::move(pkginfo),
/* .install= */ std::move(newer),
});
break;
}
case SOLVER_TRANSACTION_REINSTALLED:
{
LOG_DEBUG << "Solution: Reinstall " << pkginfo.str();
out.emplace_back(Solution::Reinstall{ std::move(pkginfo) });
break;
}
case SOLVER_TRANSACTION_DOWNGRADED:
{
auto newer = get_newer_pkginfo(id);
LOG_DEBUG << "Solution: Downgrade " << pkginfo.str() << " -> " << newer.str();
out.emplace_back(Solution::Downgrade{
/* .remove= */ std::move(pkginfo),
/* .install= */ std::move(newer),
});
break;
}
case SOLVER_TRANSACTION_ERASE:
{
LOG_DEBUG << "Solution: Remove " << pkginfo.str();
out.emplace_back(Solution::Remove{ std::move(pkginfo) });
break;
}
case SOLVER_TRANSACTION_INSTALL:
{
LOG_DEBUG << "Solution: Install " << pkginfo.str();
out.emplace_back(Solution::Install{ std::move(pkginfo) });
break;
}
case SOLVER_TRANSACTION_IGNORE:
break;
default:
LOG_WARNING << "solv::ObjTransaction case not handled: " << type;
break;
}
}
);
return { std::move(out) };
}
}

24
libmamba/src/solver/libsolv/helpers.hpp
View File

@ -8,11 +8,15 @@
#define MAMBA_SOLVER_LIBSOLV_HERLPERS
#include "mamba/core/error_handling.hpp"
#include "mamba/core/solution.hpp"
#include "mamba/solver/libsolv/parameters.hpp"
#include "mamba/specs/channel.hpp"
#include "mamba/specs/match_spec.hpp"
#include "mamba/specs/package_info.hpp"
#include "solv-cpp/pool.hpp"
#include "solv-cpp/repo.hpp"
#include "solv-cpp/solvable.hpp"
#include "solv-cpp/transaction.hpp"
namespace mamba::fs
{
@ -57,5 +61,25 @@ namespace mamba::solver::libsolv
void set_solvables_url(solv::ObjRepoView repo, const std::string& repo_url);
void add_pip_as_python_dependency(solv::ObjPool& pool, solv::ObjRepoView repo);
[[nodiscard]] auto pool_add_matchspec( //
solv::ObjPool& pool,
const specs::MatchSpec& ms,
const specs::ChannelResolveParams& params
) -> expected_t<solv::DependencyId>;
[[nodiscard]] auto pool_add_pin( //
solv::ObjPool& pool,
const specs::MatchSpec& pin_ms,
const specs::ChannelResolveParams& params
) -> expected_t<solv::ObjSolvableView>;
[[nodiscard]] auto transaction_to_solution(
const solv::ObjPool& pool,
const solv::ObjTransaction& trans,
const util::flat_set<std::string>& specs = {},
/** true to filter out specs, false to filter in specs */
bool keep_only = true
) -> Solution;
}
#endif

8
libmamba/tests/src/solv-cpp/test_solvable.cpp
View File

@ -51,7 +51,7 @@ TEST_SUITE("solv::ObjSolvable")
solv.set_url("https://conda.anaconda.org/conda-forge/linux-64");
solv.set_channel("conda-forge");
solv.set_subdir("linux-64");
solv.set_artificial(true);
solv.set_type(SolvableType::Virtualpackage);
SUBCASE("Empty without internalize")
{
@ -67,7 +67,7 @@ TEST_SUITE("solv::ObjSolvable")
CHECK_EQ(solv.url(), "");
CHECK_EQ(solv.channel(), "");
CHECK_EQ(solv.subdir(), "");
CHECK_EQ(solv.artificial(), false);
CHECK_EQ(solv.type(), SolvableType::Package);
}
SUBCASE("Internalize and get attributes")
@ -90,7 +90,7 @@ TEST_SUITE("solv::ObjSolvable")
CHECK_EQ(solv.url(), "https://conda.anaconda.org/conda-forge/linux-64");
CHECK_EQ(solv.channel(), "conda-forge");
CHECK_EQ(solv.subdir(), "linux-64");
CHECK_EQ(solv.artificial(), true);
CHECK_EQ(solv.type(), SolvableType::Virtualpackage);
SUBCASE("Override attribute")
{
@ -118,7 +118,7 @@ TEST_SUITE("solv::ObjSolvable")
CHECK_EQ(solv.url(), "");
CHECK_EQ(solv.channel(), "");
CHECK_EQ(solv.subdir(), "");
CHECK_EQ(solv.artificial(), false);
CHECK_EQ(solv.type(), SolvableType::Package);
}
SUBCASE("Add dependency")

65
libmambapy/src/libmambapy/bindings/legacy.cpp
View File

@ -310,6 +310,10 @@ bind_submodule_impl(pybind11::module_ m)
}
));
m.attr("MAMBA_NO_DEPS") = "V2 Migration: Use Solver.Flags instead";
m.attr("MAMBA_ONLY_DEPS") = "V2 Migration: Use Solver.Flags instead";
m.attr("MAMBA_FORCE_REINSTALL") = "V2 Migration: Use Solver.Flags instead";
/**************
* Bindings *
**************/
@ -441,24 +445,54 @@ bind_submodule_impl(pybind11::module_ m)
.def("find_python_version", &MTransaction::py_find_python_version)
.def("execute", &MTransaction::execute);
py::class_<MSolverProblem>(m, "SolverProblem")
.def_readwrite("type", &MSolverProblem::type)
.def_readwrite("source_id", &MSolverProblem::source_id)
.def_readwrite("target_id", &MSolverProblem::target_id)
.def_readwrite("dep_id", &MSolverProblem::dep_id)
.def_readwrite("source", &MSolverProblem::source)
.def_readwrite("target", &MSolverProblem::target)
.def_readwrite("dep", &MSolverProblem::dep)
.def_readwrite("description", &MSolverProblem::description)
.def("__str__", [](const MSolverProblem& self) { return self.description; });
py::class_<SolverProblem>(m, "SolverProblem")
.def_readwrite("type", &SolverProblem::type)
.def_readwrite("source_id", &SolverProblem::source_id)
.def_readwrite("target_id", &SolverProblem::target_id)
.def_readwrite("dep_id", &SolverProblem::dep_id)
.def_readwrite("source", &SolverProblem::source)
.def_readwrite("target", &SolverProblem::target)
.def_readwrite("dep", &SolverProblem::dep)
.def_readwrite("description", &SolverProblem::description)
.def("__str__", [](const SolverProblem& self) { return self.description; });
py::class_<MSolver::Flags>(pySolver, "Flags")
.def(py::init(
[](bool keep_dependencies, bool keep_specs, bool force_reinstall) -> MSolver::Flags
{
return {
/* .keep_dependencies= */ keep_dependencies,
/* .keep_specs= */ keep_specs,
/* .force_reinstall= */ force_reinstall,
};
}
))
.def_readwrite("keep_dependencies", &MSolver::Flags::keep_dependencies)
.def_readwrite("keep_specs", &MSolver::Flags::keep_specs)
.def_readwrite("force_reinstall", &MSolver::Flags::force_reinstall);
pySolver.def(py::init<MPool&, std::vector<std::pair<int, int>>>(), py::keep_alive<1, 2>())
.def("add_jobs", &MSolver::add_jobs)
.def("add_global_job", &MSolver::add_global_job)
.def("add_constraint", &MSolver::add_constraint)
.def("add_pin", &MSolver::add_pin)
.def("set_flags", &MSolver::py_set_libsolv_flags)
.def("set_postsolve_flags", &MSolver::py_set_postsolve_flags)
.def("set_libsolv_flags", &MSolver::py_set_libsolv_flags)
.def(
"set_flags",
[](MSolver&, const std::vector<std::pair<int, int>>&)
{
// V2 migrator
throw std::runtime_error("Use Solver.set_libsolv_flags instead.");
}
)
.def("set_flags", &MSolver::set_flags)
.def(
"set_postsolve_flags",
[](MSolver&, const std::vector<std::pair<int, int>>&)
{
// V2 migrator
throw std::runtime_error("Use Solver.set_flags with Solver.Flags object instead.");
}
)
.def("is_solved", &MSolver::is_solved)
.def("problems_to_str", &MSolver::problems_to_str)
.def("all_problems_to_str", &MSolver::all_problems_to_str)
@ -1396,11 +1430,6 @@ bind_submodule_impl(pybind11::module_ m)
.value("SOLVER_RULE_BLACK", SolverRuleinfo::SOLVER_RULE_BLACK)
.value("SOLVER_RULE_STRICT_REPO_PRIORITY", SolverRuleinfo::SOLVER_RULE_STRICT_REPO_PRIORITY);
// INSTALL FLAGS
m.attr("MAMBA_NO_DEPS") = PY_MAMBA_NO_DEPS;
m.attr("MAMBA_ONLY_DEPS") = PY_MAMBA_ONLY_DEPS;
m.attr("MAMBA_FORCE_REINSTALL") = PY_MAMBA_FORCE_REINSTALL;
// CLEAN FLAGS
m.attr("MAMBA_CLEAN_ALL") = MAMBA_CLEAN_ALL;
m.attr("MAMBA_CLEAN_INDEX") = MAMBA_CLEAN_INDEX;