folkslinux
/
mamba
mirror of https://github.com/mamba-org/mamba.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use libsolv wrappers in MPool and MRepo (#2453) 

* Use ObjPool in MPool

* Straightforward wrapper call

* MPool debug callback wrapper

* Fix format

* Use m_real_repo_key

* MRepos need not be stored

* Change MRepo::set_urls call

* Rename real_repo_key

* Awlways set solvable URL

* Rename mrepo_key

* Leaf comment note on package Info

* Ensure proper channel format in query

* Fix set url everywhere

* Rename solvable:noarch

* Fix real_repo_url in pybind

* Bump mamba tool version

* fix: prevent (possible) dangling pointer in query

* Fix url in transaction

* fix: setting url or repo from mamba

* fix: UB string_view of deallocated string

* Use solv:: interface in channel specific dep

* Use new solv:: interface in make_package_info

* Use solv:: interface in add_package_info

* Small implementation changes in MRepo

* Use wrapped reading functions

* kill MRepo::m_url

* Mark MRepo C++ function for Python deprecated

* Add ObjRepo::legacy_read_conda_repodata

* Mark more MRepo functions private

* Remove filename attributes from MRepo

* Fix Mrepo add_extra_pkg_info

* Not ignoring MRepo name

* Split url outside MRepo

* Remove MRepo ctor overload
This commit is contained in:
Antoine Prouvost 2023-05-05 11:07:21 +02:00 committed by GitHub
parent e5b30cc268
commit 15f6b6e606
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 564 additions and 743 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
libmamba/include/mamba/core/package_info.hpp
View File

@ -43,6 +43,11 @@ namespace mamba
std::string build_string = {};
std::string noarch = {};
std::size_t build_number = 0;
/**
* Could contain "conda-forge", "conda-forge/linux-64", or a url.
*
* @todo need to use a proper type for channels
*/
std::string channel = {};
std::string url = {};
std::string subdir = {};

22
libmamba/include/mamba/core/pool.hpp
View File

@ -13,12 +13,16 @@
#include <solv/pooltypes.h>
#include "mamba/core/package_info.hpp"
#include "mamba/core/repo.hpp"
namespace mamba
{
class MatchSpec;
namespace solv
{
class ObjPool;
}
/**
* Pool of solvable involved in resolving en environment.
*
@ -34,8 +38,6 @@ namespace mamba
MPool();
~MPool();
std::size_t n_solvables() const;
void set_debuglevel();
void create_whatprovides();
@ -45,18 +47,20 @@ namespace mamba
std::optional<PackageInfo> id2pkginfo(Id solv_id) const;
std::optional<std::string> dep2str(Id dep_id) const;
operator Pool*();
operator const Pool*() const;
// TODO: (TMP) This is not meant to exist but is needed for a transition period
operator ::Pool*();
operator const ::Pool*() const;
MRepo& add_repo(MRepo&& repo);
void remove_repo(Id repo_id);
// TODO: (TMP) This is not meant to be public but is needed for a transition period
solv::ObjPool& pool();
const solv::ObjPool& pool() const;
void remove_repo(::Id repo_id, bool reuse_ids);
private:
struct MPoolData;
Pool* pool();
const Pool* pool() const;
/**
* Make MPool behave like a shared_ptr (with move and copy).

121
libmamba/include/mamba/core/repo.hpp
View File

@ -8,13 +8,15 @@
#define MAMBA_CORE_REPO_HPP
#include <string>
#include <string_view>
#include <tuple>
#include <utility>
#include <vector>
#include <nlohmann/json_fwd.hpp>
#include <solv/pooltypes.h>
#include "mamba_fs.hpp"
#include "pool.hpp"
extern "C"
{
@ -22,9 +24,13 @@ extern "C"
typedef struct s_Repodata Repodata;
}
namespace fs
{
class u8path;
}
namespace mamba
{
class MPool;
class PackageInfo;
class PrefixData;
@ -34,17 +40,17 @@ namespace mamba
*/
struct RepoMetadata
{
std::string url;
std::string url = {};
std::string etag = {};
std::string mod = {};
bool pip_added = false;
std::string etag;
std::string mod;
};
inline bool operator==(const RepoMetadata& lhs, const RepoMetadata& rhs)
{
return lhs.url == rhs.url && lhs.pip_added == rhs.pip_added && lhs.etag == rhs.etag
&& lhs.mod == rhs.mod;
}
auto operator==(const RepoMetadata& lhs, const RepoMetadata& rhs) -> bool;
auto operator!=(const RepoMetadata& lhs, const RepoMetadata& rhs) -> bool;
void to_json(nlohmann::json& j, const RepoMetadata& m);
void from_json(const nlohmann::json& j, RepoMetadata& p);
/**
* A wrapper class of libsolv Repo.
@ -56,91 +62,52 @@ namespace mamba
{
public:
~MRepo();
MRepo(MPool& pool, const std::string& name, const fs::u8path& filename, const RepoMetadata& meta);
MRepo(MPool& pool, const PrefixData& prefix_data);
MRepo(MPool& pool, const std::string& name, const std::vector<PackageInfo>& uris);
MRepo(const MRepo&) = delete;
MRepo(MRepo&&) = default;
MRepo& operator=(const MRepo&) = delete;
MRepo(MRepo&&);
MRepo& operator=(MRepo&&);
MRepo& operator=(MRepo&&) = default;
void set_installed();
void set_priority(int priority, int subpriority);
void add_package_info(Repodata*, const PackageInfo& pkg_info);
void add_pip_as_python_dependency();
const fs::u8path& index_file();
Id id() const;
std::string name() const;
bool write() const;
const std::string& url() const;
Repo* repo() const;
std::tuple<int, int> priority() const;
std::size_t size() const;
bool clear(bool reuse_ids);
struct [[deprecated]] PyExtraPkgInfo
{
std::string noarch;
std::string repo_url;
};
/**
* Static constructor.
* @param pool ``libsolv`` pool wrapper
* @param name Name of the subdirectory (<channel>/<subdir>)
* @param filename Name of the index file
* @param url Subdirectory URL
*/
static MRepo&
create(MPool& pool, const std::string& name, const std::string& filename, const std::string& url);
/**
* Static constructor.
* @param pool ``libsolv`` pool wrapper
* @param name Name of the subdirectory (<channel>/<subdir>)
* @param index Path to the index file
* @param meta Metadata of the repo
*/
static MRepo&
create(MPool& pool, const std::string& name, const fs::u8path& filename, const RepoMetadata& meta);
/**
* Static constructor.
* @param pool ``libsolv`` pool wrapper
* @param prefix_data prefix data
*/
static MRepo& create(MPool& pool, const PrefixData& prefix_data);
/**
* Static constructor.
* @param pool ``libsolv`` pool wrapper
* @param name Name
* @param uris Matchspecs pointing to unique resources (URL or files)
*/
static MRepo&
create(MPool& pool, const std::string& name, const std::vector<PackageInfo>& uris);
[[deprecated]] auto py_name() const -> std::string_view;
[[deprecated]] auto py_priority() const -> std::tuple<int, int>;
[[deprecated]] auto py_clear(bool reuse_ids) -> bool;
[[deprecated]] auto py_size() const -> std::size_t;
[[deprecated]] void
py_add_extra_pkg_info(const std::map<std::string, PyExtraPkgInfo>& additional_info);
private:
MRepo(MPool& pool, const std::string& name, const std::string& filename, const std::string& url);
auto name() const -> std::string_view;
MRepo(MPool& pool, const std::string& name, const fs::u8path& filename, const RepoMetadata& meta);
void add_pip_as_python_dependency();
void clear(bool reuse_ids = true);
void load_file(const fs::u8path& filename);
void read_json(const fs::u8path& filename);
bool read_solv(const fs::u8path& filename);
void write_solv(fs::u8path path);
void add_package_info(const PackageInfo& pkg_info);
void set_solvables_url(const std::string& repo_url);
MRepo(MPool& pool, const PrefixData& prefix_data);
MPool m_pool;
MRepo(MPool& pool, const std::string& name, const std::vector<PackageInfo>& uris);
RepoMetadata m_metadata = {};
void init(MPool& pool);
bool read_file(const fs::u8path& filename);
void set_solvables_url();
fs::u8path m_json_file, m_solv_file;
std::string m_url;
RepoMetadata m_metadata;
Repo* m_repo;
::Id m_real_repo_key = 0;
::Id m_mrepo_key = 0;
::Id m_noarch_repo_key = 0;
Repo* m_repo = nullptr; // This is a view managed by libsolv pool
};
} // namespace mamba

2
libmamba/include/mamba/core/subdirdata.hpp
View File

@ -107,7 +107,7 @@ namespace mamba
bool finalize_check(const DownloadTarget& target);
bool finalize_transfer(const DownloadTarget& target);
void finalize_checks();
expected_t<MRepo&> create_repo(MPool& pool);
expected_t<MRepo> create_repo(MPool& pool);
private:

2
libmamba/include/mamba/core/transaction.hpp
View File

@ -165,8 +165,6 @@ namespace mamba
History::UserRequest m_history_entry;
Transaction* m_transaction;
::Id m_real_repo_key = 0; // Must match that in ``MRepo``
::Id m_mrepo_key = 0; // Must match that in ``MRepo``
std::vector<MatchSpec> m_requested_specs;

8
libmamba/src/api/channel_loader.cpp
View File

@ -15,9 +15,9 @@
namespace mamba
{
namespace detail
namespace
{
MRepo& create_repo_from_pkgs_dir(MPool& pool, const fs::u8path& pkgs_dir)
MRepo create_repo_from_pkgs_dir(MPool& pool, const fs::u8path& pkgs_dir)
{
if (!fs::exists(pkgs_dir))
{
@ -39,7 +39,7 @@ namespace mamba
}
prefix_data.load_single_record(repodata_record_json);
}
return MRepo::create(pool, prefix_data);
return MRepo(pool, prefix_data);
}
}
@ -136,7 +136,7 @@ namespace mamba
LOG_INFO << "Creating repo from pkgs_dir for offline";
for (const auto& c : ctx.pkgs_dirs)
{
detail::create_repo_from_pkgs_dir(pool, c);
create_repo_from_pkgs_dir(pool, c);
}
}
std::string prev_channel;

6
libmamba/src/api/install.cpp
View File

@ -492,7 +492,7 @@ namespace mamba
prefix_data.add_packages(get_virtual_packages());
MRepo::create(pool, prefix_data);
MRepo(pool, prefix_data);
MSolver solver(
pool,
@ -611,8 +611,8 @@ namespace mamba
MultiPackageCache pkg_caches(ctx.pkgs_dirs);
prefix_data.add_packages(get_virtual_packages());
MRepo::create(pool, prefix_data); // Potentially re-alloc (moves in memory) Solvables
// in the pool
MRepo(pool, prefix_data); // Potentially re-alloc (moves in memory) Solvables
// in the pool
std::vector<detail::other_pkg_mgr_spec> others;
// Note that the Transaction will gather the Solvables,

2
libmamba/src/api/remove.cpp
View File

@ -83,7 +83,7 @@ namespace mamba
PrefixData& prefix_data = exp_prefix_data.value();
MPool pool;
MRepo::create(pool, prefix_data);
MRepo(pool, prefix_data);
const fs::u8path pkgs_dirs(ctx.prefix_params.root_prefix / "pkgs");
MultiPackageCache package_caches({ pkgs_dirs });

3
libmamba/src/api/repoquery.cpp
View File

@ -11,6 +11,7 @@
#include "mamba/api/repoquery.hpp"
#include "mamba/core/package_cache.hpp"
#include "mamba/core/prefix_data.hpp"
#include "mamba/core/repo.hpp"
#include "mamba/core/util_string.hpp"
namespace mamba
@ -43,7 +44,7 @@ namespace mamba
throw std::runtime_error(exp_prefix_data.error().what());
}
PrefixData& prefix_data = exp_prefix_data.value();
MRepo::create(pool, prefix_data);
MRepo(pool, prefix_data);
if (format != QueryResultFormat::kJSON)
{
Console::stream() << "Loaded current active prefix: "

2
libmamba/src/api/update.cpp
View File

@ -66,7 +66,7 @@ namespace mamba
prefix_data.add_packages(get_virtual_packages());
MRepo::create(pool, prefix_data);
MRepo(pool, prefix_data);
MSolver solver(
pool,

276
libmamba/src/core/pool.cpp
View File

@ -4,7 +4,7 @@
//
// The full license is in the file LICENSE, distributed with this software.
#include <list>
#include <string_view>
#include <fmt/format.h>
#include <solv/evr.h>
@ -25,125 +25,97 @@ extern "C" // Incomplete header
#include "mamba/core/util_string.hpp"
#include "mamba/util/cast.hpp"
#include "mamba/util/compare.hpp"
#include "solv-cpp/pool.hpp"
#include "solv-cpp/queue.hpp"
namespace mamba
{
namespace
{
void libsolv_debug_callback(Pool* /*pool*/, void* userptr, int type, const char* str)
{
auto* dbg = reinterpret_cast<std::pair<spdlog::logger*, std::string>*>(userptr);
dbg->second += str;
if (dbg->second.size() == 0 || dbg->second.back() != '\n')
{
return;
}
auto log = Console::hide_secrets(dbg->second);
if (type & SOLV_FATAL || type & SOLV_ERROR)
{
dbg->first->error(log);
}
else if (type & SOLV_WARN)
{
dbg->first->warn(log);
}
else if (Context::instance().output_params.verbosity > 2)
{
dbg->first->info(log);
}
dbg->second.clear();
}
void libsolv_delete_pool(::Pool* pool)
{
LOG_INFO << "Freeing pool.";
pool_free(pool);
}
}
struct MPool::MPoolData
{
MPoolData()
: pool(pool_create(), &libsolv_delete_pool)
{
}
std::unique_ptr<Pool, decltype(&libsolv_delete_pool)> pool;
std::pair<spdlog::logger*, std::string> debug_logger = {};
std::list<MRepo> repo_list = {};
solv::ObjPool pool = {};
};
MPool::MPool()
: m_data(std::make_shared<MPoolData>())
{
pool_setdisttype(pool(), DISTTYPE_CONDA);
pool().set_disttype(DISTTYPE_CONDA);
set_debuglevel();
}
MPool::~MPool() = default;
Pool* MPool::pool()
solv::ObjPool& MPool::pool()
{
return m_data->pool.get();
return m_data->pool;
}
const Pool* MPool::pool() const
const solv::ObjPool& MPool::pool() const
{
return m_data->pool.get();
}
std::size_t MPool::n_solvables() const
{
return util::safe_num_cast<std::size_t>(pool()->nsolvables);
return m_data->pool;
}
void MPool::set_debuglevel()
{
// ensure that debug logging goes to stderr as to not interfere with stdout json output
pool()->debugmask |= SOLV_DEBUG_TO_STDERR;
pool().raw()->debugmask |= SOLV_DEBUG_TO_STDERR;
if (Context::instance().output_params.verbosity > 2)
{
pool_setdebuglevel(pool(), Context::instance().output_params.verbosity - 1);
auto logger = spdlog::get("libsolv");
m_data->debug_logger.first = logger.get();
pool_setdebugcallback(pool(), &libsolv_debug_callback, &(m_data->debug_logger));
pool_setdebuglevel(pool().raw(), Context::instance().output_params.verbosity - 1);
pool().set_debug_callback(
[logger = spdlog::get("libsolv")](::Pool*, int type, std::string_view msg) noexcept
{
if (msg.size() == 0 || msg.back() != '\n')
{
return;
}
auto log = Console::hide_secrets(msg);
if (type & SOLV_FATAL || type & SOLV_ERROR)
{
logger->error(log);
}
else if (type & SOLV_WARN)
{
logger->warn(log);
}
else if (Context::instance().output_params.verbosity > 2)
{
logger->info(log);
}
}
);
}
}
void MPool::create_whatprovides()
{
pool_createwhatprovides(pool());
pool().create_whatprovides();
}
MPool::operator Pool*()
{
return pool();
return pool().raw();
}
MPool::operator const Pool*() const
{
return pool();
return pool().raw();
}
std::vector<Id> MPool::select_solvables(Id matchspec, bool sorted) const
{
solv::ObjQueue job = { SOLVER_SOLVABLE_PROVIDES, matchspec };
solv::ObjQueue solvables = {};
selection_solvables(const_cast<Pool*>(pool()), job.raw(), solvables.raw());
auto solvables = pool().select_solvables({ SOLVER_SOLVABLE_PROVIDES, matchspec });
if (sorted)
{
std::sort(
solvables.begin(),
solvables.end(),
[this](Id a, Id b)
[pool_ptr = pool().raw()](Id a, Id b)
{
Solvable* sa = pool_id2solvable(pool(), a);
Solvable* sb = pool_id2solvable(pool(), b);
return (pool_evrcmp(this->pool(), sa->evr, sb->evr, EVRCMP_COMPARE) > 0);
Solvable* sa = pool_id2solvable(pool_ptr, a);
Solvable* sb = pool_id2solvable(pool_ptr, b);
return (pool_evrcmp(pool_ptr, sa->evr, sb->evr, EVRCMP_COMPARE) > 0);
}
);
}
@ -176,40 +148,50 @@ namespace mamba
return false;
}
::Id add_channel_specific_matchspec(::Pool* pool, const MatchSpec& ms)
/**
* Add function to handle matchspec while parsing is done by libsolv.
*/
auto add_channel_specific_matchspec(solv::ObjPool& pool, const MatchSpec& ms)
-> solv::DependencyId
{
// Poor man's ms repr to match waht the user provided
std::string const repr = fmt::format("{}::{}", ms.channel, ms.conda_build_form());
// Already added, return that id
if (::Id repr_id = pool_str2id(pool, repr.c_str(), /* .create= */ false); repr_id != 0)
if (const auto maybe_id = pool.find_string(repr))
{
return repr_id;
return maybe_id.value();
}
solv::ObjQueue selected_pkgs;
// conda_build_form does **NOT** contain the channel info
::Id match = pool_conda_matchspec(pool, ms.conda_build_form().c_str());
solv::DependencyId const match = pool_conda_matchspec(
pool.raw(),
ms.conda_build_form().c_str()
);
const Channel& c = make_channel(ms.channel);
::Id const m_mrepo_key = pool_str2id(pool, "solvable:mrepo_url", 1);
for (Id* wp = pool_whatprovides_ptr(pool, match); *wp; wp++)
{
auto* const s = pool_id2solvable(pool, *wp);
const char* s_url = solvable_lookup_str(s, m_mrepo_key);
if ((s_url != nullptr) && channel_match(make_channel(s_url), c))
solv::ObjQueue selected_pkgs = {};
pool.for_each_whatprovides(
match,
[&](solv::ObjSolvableViewConst s)
{
selected_pkgs.push_back(*wp);
// TODO this does not work with s.url(), we need to proper channel class
// to properly manage this.
auto repo = solv::ObjRepoView(*s.raw()->repo);
// TODO make_channel should disapear avoiding conflict here
auto const url = std::string(repo.url());
if (channel_match(make_channel(url), c))
{
selected_pkgs.push_back(s.id());
}
}
}
::Id const repr_id = pool_str2id(pool, repr.c_str(), /* .create= */ true);
::Id const offset = pool_queuetowhatprovides(pool, selected_pkgs.raw());
);
solv::StringId const repr_id = pool.add_string(repr);
::Id const offset = pool_queuetowhatprovides(pool.raw(), selected_pkgs.raw());
// FRAGILE This get deleted when calling ``pool_createwhatprovides`` so care
// must be taken to do it before
// TODO investigate namespace providers
pool_set_whatprovides(pool, repr_id, offset);
pool_set_whatprovides(pool.raw(), repr_id, offset);
return repr_id;
}
}
@ -219,7 +201,7 @@ namespace mamba
::Id id = 0;
if (ms.channel.empty())
{
id = pool_conda_matchspec(pool(), ms.conda_build_form().c_str());
id = pool_conda_matchspec(pool().raw(), ms.conda_build_form().c_str());
}
else
{
@ -236,94 +218,61 @@ namespace mamba
namespace
{
auto make_package_info(::Solvable& s) -> PackageInfo
auto make_package_info(const solv::ObjPool& pool, solv::ObjSolvableViewConst s) -> PackageInfo
{
// Note: this function (especially the checksum part) is NOT YET threadsafe!
Pool* pool = s.repo->pool;
Id check_type;
PackageInfo out = {};
out.name = pool_id2str(pool, s.name);
out.version = pool_id2str(pool, s.evr);
out.build_string = raw_str_or_empty(solvable_lookup_str(&s, SOLVABLE_BUILDFLAVOR));
if (const char* str = solvable_lookup_str(&s, SOLVABLE_BUILDVERSION); str != nullptr)
{
out.build_number = std::stoull(str);
}
out.name = s.name();
out.version = s.version();
out.build_string = s.build_string();
out.noarch = s.noarch();
out.build_number = s.build_number();
out.channel = s.channel();
out.url = s.url();
out.subdir = s.subdir();
out.fn = s.file_name();
out.license = s.license();
out.size = s.size();
out.timestamp = s.timestamp();
out.md5 = s.md5();
out.sha256 = s.sha256();
static ::Id real_repo_key = pool_str2id(pool, "solvable:real_repo_url", 1);
if (const char* str = solvable_lookup_str(&s, real_repo_key); str != nullptr)
const auto dep_to_str = [&pool](solv::DependencyId id)
{ return pool.dependency_to_string(id); };
{
out.url = str;
out.channel = make_channel(out.url).canonical_name();
const auto deps = s.dependencies();
out.depends.reserve(deps.size());
std::transform(deps.cbegin(), deps.cend(), std::back_inserter(out.depends), dep_to_str);
}
else
{
if (!s.repo || strcmp(s.repo->name, "__explicit_specs__") == 0)
{
out.url = solvable_lookup_location(&s, 0);
out.channel = make_channel(out.url).canonical_name();
}
else
{
out.channel = s.repo->name; // note this can and should be <unknown> when e.g.
// installing from a tarball
out.url = fmt::format(
"{}/{}",
out.channel,
raw_str_or_empty(solvable_lookup_str(&s, SOLVABLE_MEDIAFILE))
);
}
const auto cons = s.constraints();
out.constrains.reserve(cons.size());
std::transform(cons.cbegin(), cons.cend(), std::back_inserter(out.constrains), dep_to_str);
}
out.subdir = raw_str_or_empty(solvable_lookup_str(&s, SOLVABLE_MEDIADIR));
out.fn = raw_str_or_empty(solvable_lookup_str(&s, SOLVABLE_MEDIAFILE));
out.license = raw_str_or_empty(solvable_lookup_str(&s, SOLVABLE_LICENSE));
out.size = solvable_lookup_num(&s, SOLVABLE_DOWNLOADSIZE, 0);
out.timestamp = solvable_lookup_num(&s, SOLVABLE_BUILDTIME, 0);
out.md5 = raw_str_or_empty(solvable_lookup_checksum(&s, SOLVABLE_PKGID, &check_type));
out.sha256 = raw_str_or_empty(solvable_lookup_checksum(&s, SOLVABLE_CHECKSUM, &check_type)
);
out.signatures = raw_str_or_empty(solvable_lookup_str(&s, SIGNATURE_DATA));
if (out.signatures.empty())
{
out.signatures = "{}";
const auto id_to_str = [&pool](solv::StringId id)
{ return std::string(pool.get_string(id)); };
auto feats = s.track_features();
out.track_features.reserve(feats.size());
std::transform(
feats.begin(),
feats.end(),
std::back_inserter(out.track_features),
id_to_str
);
}
solv::ObjQueue q = {};
auto dep2str = [&pool](Id id) { return pool_dep2str(pool, id); };
if (!solvable_lookup_deparray(&s, SOLVABLE_REQUIRES, q.raw(), -1))
{
out.defaulted_keys.insert("depends");
}
out.depends.reserve(q.size());
std::transform(q.begin(), q.end(), std::back_inserter(out.depends), dep2str);
q.clear();
if (!solvable_lookup_deparray(&s, SOLVABLE_CONSTRAINS, q.raw(), -1))
{
out.defaulted_keys.insert("constrains");
}
out.constrains.reserve(q.size());
std::transform(q.begin(), q.end(), std::back_inserter(out.constrains), dep2str);
q.clear();
auto id2str = [&pool](Id id) { return pool_id2str(pool, id); };
solvable_lookup_idarray(&s, SOLVABLE_TRACK_FEATURES, q.raw());
std::transform(q.begin(), q.end(), std::back_inserter(out.track_features), id2str);
return out;
}
}
std::optional<PackageInfo> MPool::id2pkginfo(Id solv_id) const
{
if (solv_id == 0 || util::cmp_greater_equal(solv_id, n_solvables()))
if (const auto solv = pool().get_solvable(solv_id))
{
return std::nullopt;
return { make_package_info(pool(), solv.value()) };
}
return { make_package_info(*pool_id2solvable(pool(), solv_id)) };
return std::nullopt;
}
std::optional<std::string> MPool::dep2str(Id dep_id) const
@ -333,18 +282,11 @@ namespace mamba
return std::nullopt;
}
// Not const because might alloctmp space
return pool_dep2str(const_cast<::Pool*>(pool()), dep_id);
return pool_dep2str(const_cast<::Pool*>(pool().raw()), dep_id);
}
MRepo& MPool::add_repo(MRepo&& repo)
void MPool::remove_repo(::Id repo_id, bool reuse_ids)
{
m_data->repo_list.push_back(std::move(repo));
return m_data->repo_list.back();
pool().remove_repo(repo_id, reuse_ids);
}
void MPool::remove_repo(Id repo_id)
{
m_data->repo_list.remove_if([repo_id](const MRepo& repo) { return repo_id == repo.id(); });
}
} // namespace mamba

36
libmamba/src/core/query.cpp
View File

@ -411,6 +411,16 @@ namespace mamba
return table(out, { "Name", "Version", "Build", "Channel" });
}
namespace
{
/** Remove potential subdir from channel name (not url!). */
auto cut_subdir(std::string_view str) -> std::string
{
return split(str, "/", 1).front(); // Has at least one element
}
}
std::ostream& query_result::table(std::ostream& out, const std::vector<std::string>& fmt) const
{
if (m_pkg_id_list.empty())
@ -457,7 +467,7 @@ namespace mamba
}
else if (cmd == "Channel")
{
row.push_back(cut_repo_name(pkg.channel));
row.push_back(cut_subdir(cut_repo_name(pkg.channel)));
}
else if (cmd == "Depends")
{
@ -630,16 +640,30 @@ namespace mamba
j["result"]["pkgs"] = nlohmann::json::array();
for (size_t i = 0; i < m_pkg_id_list.size(); ++i)
{
j["result"]["pkgs"].push_back(m_dep_graph.node(m_pkg_id_list[i]).json_record());
auto pkg_info_json = m_dep_graph.node(m_pkg_id_list[i]).json_record();
// We want the cannonical channel name here.
// We do not know what is in the `channel` field so we need to make sure.
// This is most likely legacy and should be updated on the next major release.
pkg_info_json["channel"] = cut_subdir(cut_repo_name(pkg_info_json["channel"]));
j["result"]["pkgs"].push_back(std::move(pkg_info_json));
}
if (m_type != QueryType::kSEARCH && !m_pkg_id_list.empty())
{
bool has_root = !m_dep_graph.successors(0).empty();
j["result"]["graph_roots"] = nlohmann::json::array();
j["result"]["graph_roots"].push_back(
has_root ? m_dep_graph.node(0).json_record() : nlohmann::json(m_query)
);
if (!m_dep_graph.successors(0).empty())
{
auto pkg_info_json = m_dep_graph.node(0).json_record();
// We want the cannonical channel name here.
// We do not know what is in the `channel` field so we need to make sure.
// This is most likely legacy and should be updated on the next major release.
pkg_info_json["channel"] = cut_subdir(cut_repo_name(pkg_info_json["channel"]));
j["result"]["graph_roots"].push_back(std::move(pkg_info_json));
}
else
{
j["result"]["graph_roots"].push_back(nlohmann::json(m_query));
}
}
return j;
}

659
libmamba/src/core/repo.cpp
View File

@ -4,6 +4,11 @@
//
// The full license is in the file LICENSE, distributed with this software.
#include <algorithm>
#include <array>
#include <tuple>
#include <nlohmann/json.hpp>
#include <solv/repo.h>
#include <solv/repo_solv.h>
#include <solv/repo_write.h>
@ -16,71 +21,104 @@ extern "C" // Incomplete header
#include "mamba/core/channel.hpp"
#include "mamba/core/context.hpp"
#include "mamba/core/mamba_fs.hpp"
#include "mamba/core/output.hpp"
#include "mamba/core/package_info.hpp"
#include "mamba/core/pool.hpp"
#include "mamba/core/prefix_data.hpp"
#include "mamba/core/repo.hpp"
#include "mamba/core/util_string.hpp"
#include "solv-cpp/pool.hpp"
#include "solv-cpp/repo.hpp"
#define MAMBA_TOOL_VERSION "1.1"
#define MAMBA_TOOL_VERSION "1.3"
#define MAMBA_SOLV_VERSION MAMBA_TOOL_VERSION "_" LIBSOLV_VERSION_STRING
namespace mamba
{
const char* mamba_tool_version()
namespace
{
const size_t bufferSize = 30;
static char MTV[bufferSize];
MTV[0] = '\0';
snprintf(MTV, bufferSize, MAMBA_SOLV_VERSION);
return MTV;
auto attrs(const RepoMetadata& m)
{
return std::tie(m.url, m.etag, m.mod, m.pip_added);
}
}
MRepo::MRepo(MPool& pool, const std::string& /*name*/, const fs::u8path& index, const RepoMetadata& metadata)
: m_url(rsplit(metadata.url, "/", 1)[0])
auto operator==(const RepoMetadata& lhs, const RepoMetadata& rhs) -> bool
{
return attrs(lhs) == attrs(rhs);
}
auto operator!=(const RepoMetadata& lhs, const RepoMetadata& rhs) -> bool
{
return !(lhs == rhs);
}
void to_json(nlohmann::json& j, const RepoMetadata& m)
{
j["url"] = m.url;
j["etag"] = m.etag;
j["mod"] = m.mod;
j["pip_added"] = m.pip_added;
}
void from_json(const nlohmann::json& j, RepoMetadata& m)
{
m.url = j.value("url", m.url);
m.etag = j.value("etag", m.etag);
m.mod = j.value("mod", m.mod);
m.pip_added = j.value("pip_added", m.pip_added);
}
namespace
{
// Keeping the solv-cpp header private
auto srepo(const MRepo& r) -> solv::ObjRepoViewConst
{
return solv::ObjRepoViewConst{ *const_cast<const ::Repo*>(r.repo()) };
}
auto srepo(MRepo& r) -> solv::ObjRepoView
{
return solv::ObjRepoView{ *r.repo() };
}
}
MRepo::MRepo(MPool& pool, const std::string& name, const fs::u8path& index, const RepoMetadata& metadata)
: m_pool(pool)
, m_metadata(metadata)
{
m_repo = repo_create(pool, m_url.c_str());
init(pool);
read_file(index);
}
MRepo::MRepo(MPool& pool, const std::string& name, const std::string& index, const std::string& url)
: m_url(url)
, m_repo(repo_create(pool, name.c_str()))
{
init(pool);
read_file(index);
auto [_, repo] = pool.pool().add_repo(name);
m_repo = repo.raw();
repo.set_url(m_metadata.url);
load_file(index);
set_solvables_url(m_metadata.url);
repo.internalize();
}
MRepo::MRepo(MPool& pool, const std::string& name, const std::vector<PackageInfo>& package_infos)
: m_repo(repo_create(pool, name.c_str()))
: m_pool(pool)
{
init(pool);
int flags = 0;
Repodata* data;
data = repo_add_repodata(m_repo, flags);
auto [_, repo] = pool.pool().add_repo(name);
m_repo = repo.raw();
for (auto& info : package_infos)
{
add_package_info(data, info);
add_package_info(info);
}
repodata_internalize(data);
repo.internalize();
}
MRepo::MRepo(MPool& pool, const PrefixData& prefix_data)
: m_repo(repo_create(pool, "installed"))
: m_pool(pool)
{
init(pool);
int flags = 0;
Repodata* data;
data = repo_add_repodata(m_repo, flags);
auto [repo_id, repo] = pool.pool().add_repo("installed");
m_repo = repo.raw();
for (auto& [name, record] : prefix_data.records())
{
add_package_info(data, record);
add_package_info(record);
}
if (Context::instance().add_pip_as_python_dependency)
@ -88,73 +126,31 @@ namespace mamba
add_pip_as_python_dependency();
}
repodata_internalize(data);
set_installed();
repo.internalize();
pool.pool().set_installed_repo(repo_id);
}
void MRepo::init(MPool& pool)
{
m_real_repo_key = pool_str2id(pool, "solvable:real_repo_url", 1);
m_mrepo_key = pool_str2id(pool, "solvable:mrepo_url", 1);
m_noarch_repo_key = pool_str2id(pool, "solvable:noarch_type", 1);
}
void MRepo::set_solvables_url()
void MRepo::set_solvables_url(const std::string& repo_url)
{
// WARNING cannot call ``url()`` at this point because it has not been internalized.
// Setting the channel url on where the solvable so that we can retrace
// where it came from
Id id = 0;
Solvable* s = nullptr;
FOR_REPO_SOLVABLES(m_repo, id, s)
{
solvable_set_str(s, m_mrepo_key, this->url().c_str());
}
}
MRepo::~MRepo()
{
if (m_repo)
{
repo_free(m_repo, 1);
}
// not sure if reuse_ids is useful here
// repo will be freed with pool as well though
// maybe explicitly free pool for faster repo deletion as well
// TODO this is actually freed with the pool, and calling it here will cause
// segfaults. need to find a more clever way to do this. repo_free(m_repo,
// /*reuse_ids*/1);
}
MRepo::MRepo(MRepo&& rhs)
: m_json_file(std::move(rhs.m_json_file))
, m_solv_file(std::move(rhs.m_solv_file))
, m_url(std::move(rhs.m_url))
, m_metadata(std::move(rhs.m_metadata))
, m_repo(rhs.m_repo)
, m_real_repo_key(rhs.m_real_repo_key)
, m_mrepo_key(rhs.m_mrepo_key)
, m_noarch_repo_key(rhs.m_noarch_repo_key)
{
rhs.m_repo = nullptr;
}
MRepo& MRepo::operator=(MRepo&& rhs)
{
using std::swap;
swap(m_json_file, rhs.m_json_file);
swap(m_solv_file, rhs.m_solv_file);
swap(m_url, rhs.m_url);
swap(m_metadata, rhs.m_metadata);
swap(m_repo, rhs.m_repo);
swap(m_real_repo_key, rhs.m_real_repo_key);
swap(m_mrepo_key, rhs.m_mrepo_key);
swap(m_noarch_repo_key, rhs.m_noarch_repo_key);
return *this;
srepo(*this).for_each_solvable(
[&](solv::ObjSolvableView s)
{
// The solvable url, this is not set in libsolv parsing so we set it manually
// while we still rely on libsolv for parsing
s.set_url(fmt::format("{}/{}", repo_url, s.file_name()));
// The name of the channel where it came from, may be different from repo name
// for instance with the installed repo
s.set_channel(repo_url);
}
);
}
void MRepo::set_installed()
{
pool_set_installed(m_repo->pool, m_repo);
m_pool.pool().set_installed_repo(srepo(*this).id());
}
void MRepo::set_priority(int priority, int subpriority)
@ -163,81 +159,61 @@ namespace mamba
m_repo->subpriority = subpriority;
}
void MRepo::add_package_info(Repodata* data, const PackageInfo& info)
void MRepo::add_package_info(const PackageInfo& info)
{
// TODO missing track_feature
LOG_INFO << "Adding package record to repo " << info.name;
Pool* pool = m_repo->pool;
Id handle = repo_add_solvable(m_repo);
Solvable* s = pool_id2solvable(pool, handle);
solvable_set_str(s, SOLVABLE_BUILDVERSION, std::to_string(info.build_number).c_str());
solvable_add_poolstr_array(s, SOLVABLE_BUILDFLAVOR, info.build_string.c_str());
solvable_set_str(s, SOLVABLE_NAME, info.name.c_str());
solvable_set_str(s, SOLVABLE_EVR, info.version.c_str());
solvable_set_num(s, SOLVABLE_DOWNLOADSIZE, info.size);
// No ``solvable_xxx`` equivalent
repodata_set_checksum(data, handle, SOLVABLE_PKGID, REPOKEY_TYPE_MD5, info.md5.c_str());
auto [id, solv] = srepo(*this).add_solvable();
solvable_set_str(s, m_real_repo_key, info.url.c_str());
solvable_set_str(s, m_mrepo_key, url().c_str());
if (!info.noarch.empty())
{
solvable_set_str(s, m_noarch_repo_key, info.noarch.c_str());
}
// No ``solvable_xxx`` equivalent
repodata_set_location(data, handle, 0, info.subdir.c_str(), info.fn.c_str());
repodata_set_checksum(data, handle, SOLVABLE_CHECKSUM, REPOKEY_TYPE_SHA256, info.sha256.c_str());
if (!info.depends.empty())
{
for (std::string dep : info.depends)
{
Id dep_id = pool_conda_matchspec(pool, dep.c_str());
if (dep_id)
{
s->requires = repo_addid_dep(m_repo, s->requires, dep_id, 0);
}
}
}
if (!info.constrains.empty())
{
for (std::string cst : info.constrains)
{
Id constrains_id = pool_conda_matchspec(pool, cst.c_str());
if (constrains_id)
{
solvable_add_idarray(s, SOLVABLE_CONSTRAINS, constrains_id);
}
}
}
s->provides = repo_addid_dep(
m_repo,
s->provides,
pool_rel2id(pool, s->name, s->evr, REL_EQ, /* create= */ 1),
0
solv.set_name(info.name);
solv.set_version(info.version);
solv.set_build_string(info.build_string);
solv.set_noarch(info.noarch);
solv.set_build_number(info.build_number);
solv.set_channel(info.channel);
solv.set_url(info.url);
solv.set_subdir(info.subdir);
solv.set_file_name(info.fn);
solv.set_license(info.license);
solv.set_size(info.size);
// TODO conda timestamp are not Unix timestamp.
// Libsolv normalize them this way, we need to do the same here otherwise the current
// package may get arbitrary priority.
solv.set_timestamp(
(info.timestamp > 253402300799ULL) ? (info.timestamp / 1000) : info.timestamp
);
solv.set_md5(info.md5);
solv.set_sha256(info.sha256);
repo_internalize(m_repo);
for (const auto& dep : info.depends)
{
// TODO pool's matchspec2id
solv::DependencyId const dep_id = pool_conda_matchspec(m_pool, dep.c_str());
assert(dep_id);
solv.add_dependency(dep_id);
}
for (const auto& cons : info.constrains)
{
// TODO pool's matchspec2id
solv::DependencyId const dep_id = pool_conda_matchspec(m_pool, cons.c_str());
assert(dep_id);
solv.add_constraint(dep_id);
}
solv.add_track_features(info.track_features);
solv.add_self_provide();
}
auto MRepo::name() const -> std::string_view
{
return srepo(*this).name();
}
Id MRepo::id() const
{
return m_repo->repoid;
}
std::string MRepo::name() const
{
return m_repo->name ? m_repo->name : "";
}
const std::string& MRepo::url() const
{
return m_url;
return srepo(*this).id();
}
Repo* MRepo::repo() const
@ -245,205 +221,110 @@ namespace mamba
return m_repo;
}
std::tuple<int, int> MRepo::priority() const
{
return std::make_tuple(m_repo->priority, m_repo->subpriority);
}
std::size_t MRepo::size() const
{
return static_cast<std::size_t>(m_repo->nsolvables);
}
const fs::u8path& MRepo::index_file()
{
return m_json_file;
}
void MRepo::add_pip_as_python_dependency()
{
Id pkg_id;
Solvable* pkg_s;
Id python = pool_str2id(m_repo->pool, "python", 0);
Id pip_dep = pool_conda_matchspec(m_repo->pool, "pip");
Id pip = pool_str2id(m_repo->pool, "pip", 0);
Id python_dep = pool_conda_matchspec(m_repo->pool, "python");
FOR_REPO_SOLVABLES(m_repo, pkg_id, pkg_s)
{
if (pkg_s->name == python)
solv::DependencyId const python_id = pool_conda_matchspec(m_pool, "python");
solv::DependencyId const pip_id = pool_conda_matchspec(m_pool, "pip");
srepo(*this).for_each_solvable(
[&](solv::ObjSolvableView s)
{
const char* version = pool_id2str(m_repo->pool, pkg_s->evr);
if (version && version[0] >= '2')
if (s.name() == "python")
{
pkg_s->requires = repo_addid_dep(m_repo, pkg_s->requires, pip_dep, 0);
if (!s.version().empty() && (s.version()[0] >= '2'))
{
s.add_dependency(pip_id);
}
}
if (s.name() == "pip")
{
auto deps = s.dependencies();
deps.insert(deps.begin(), python_id); // TODO is this right?
s.set_dependencies(std::move(deps));
}
}
if (pkg_s->name == pip)
{
pkg_s->requires = repo_addid_dep(
m_repo,
pkg_s->requires,
python_dep,
SOLVABLE_PREREQMARKER
);
}
);
}
void MRepo::read_json(const fs::u8path& filename)
{
LOG_INFO << "Reading repodata.json file " << filename << " for repo " << name();
// TODO make this as part of options of the repo/pool
const int flags = Context::instance().use_only_tar_bz2 ? CONDA_ADD_USE_ONLY_TAR_BZ2 : 0;
srepo(*this).legacy_read_conda_repodata(filename, flags);
}
bool MRepo::read_solv(const fs::u8path& filename)
{
LOG_INFO << "Attempting to read libsolv solv file " << filename << " for repo " << name();
auto repo = srepo(*this);
auto lock = LockFile(filename);
repo.read(filename);
const auto read_metadata = RepoMetadata{
/* .url= */ std::string(repo.url()),
/* .etag= */ std::string(repo.etag()),
/* .mod= */ std::string(repo.mod()),
/* .pip_added= */ repo.pip_added(),
};
const auto tool_version = repo.tool_version();
{
auto j = nlohmann::json(m_metadata);
j["tool_version"] = tool_version;
LOG_INFO << "Expecting solv metadata : " << j.dump();
}
{
auto j = nlohmann::json(read_metadata);
j["tool_version"] = tool_version;
LOG_INFO << "Loaded solv metadata : " << j.dump();
}
if ((tool_version != std::string_view(MAMBA_SOLV_VERSION))
|| (read_metadata == RepoMetadata{}) || (read_metadata != m_metadata))
{
LOG_INFO << "Metadata from solv are NOT valid, canceling solv file load";
repo.clear(/* reuse_ids= */ false);
return false;
}
LOG_INFO << "Metadata from solv are valid, loading successful";
return true;
}
MRepo&
MRepo::create(MPool& pool, const std::string& name, const std::string& filename, const std::string& url)
{
return pool.add_repo(MRepo(pool, name, filename, url));
}
MRepo&
MRepo::create(MPool& pool, const std::string& name, const fs::u8path& filename, const RepoMetadata& meta)
{
return pool.add_repo(MRepo(pool, name, filename, meta));
}
MRepo& MRepo::create(MPool& pool, const PrefixData& prefix_data)
{
return pool.add_repo(MRepo(pool, prefix_data));
}
MRepo& MRepo::create(MPool& pool, const std::string& name, const std::vector<PackageInfo>& uris)
{
return pool.add_repo(MRepo(pool, name, uris));
}
bool MRepo::read_file(const fs::u8path& filename)
void MRepo::load_file(const fs::u8path& filename)
{
auto repo = srepo(*this);
bool is_solv = filename.extension() == ".solv";
fs::u8path filename_wo_extension;
fs::u8path solv_file = filename;
fs::u8path json_file = filename;
if (is_solv)
{
m_solv_file = filename;
m_json_file = filename;
m_json_file.replace_extension("json");
json_file.replace_extension("json");
}
else
{
m_json_file = filename;
m_solv_file = filename;
m_solv_file.replace_extension("solv");
solv_file.replace_extension("solv");
}
LOG_INFO << "Reading cache files '" << (filename.parent_path() / filename).string()
<< ".*' for repo index '" << m_repo->name << "'";
<< ".*' for repo index '" << name() << "'";
if (is_solv)
{
auto lock = LockFile(m_solv_file);
#ifdef _WIN32
auto fp = _wfopen(m_solv_file.wstring().c_str(), L"rb");
#else
auto fp = fopen(m_solv_file.string().c_str(), "rb");
#endif
if (!fp)
const auto lock = LockFile(solv_file);
const bool read = read_solv(solv_file);
if (read)
{
throw std::runtime_error("Could not open repository file " + filename.string());
return;
}
LOG_INFO << "Attempt load from solv " << m_solv_file;
int ret = repo_add_solv(m_repo, fp, 0);
if (ret != 0)
{
LOG_ERROR << "Could not load SOLV file, falling back to JSON ("
<< pool_errstr(m_repo->pool) << ")";
}
else
{
auto* repodata = repo_last_repodata(m_repo);
if (!repodata)
{
LOG_ERROR << "Could not find valid repodata attached to solv file";
}
else
{
Id url_id = pool_str2id(m_repo->pool, "mamba:url", 1);
Id etag_id = pool_str2id(m_repo->pool, "mamba:etag", 1);
Id mod_id = pool_str2id(m_repo->pool, "mamba:mod", 1);
Id pip_added_id = pool_str2id(m_repo->pool, "mamba:pip_added", 1);
static constexpr auto failure = std::numeric_limits<unsigned long long>::max();
const char* url = repodata_lookup_str(repodata, SOLVID_META, url_id);
const auto pip_added = repodata_lookup_num(
repodata,
SOLVID_META,
pip_added_id,
failure
);
const char* etag = repodata_lookup_str(repodata, SOLVID_META, etag_id);
const char* mod = repodata_lookup_str(repodata, SOLVID_META, mod_id);
const char* tool_version = repodata_lookup_str(
repodata,
SOLVID_META,
REPOSITORY_TOOLVERSION
);
LOG_INFO << "Metadata solv file: " << url << " " << pip_added << " " << etag
<< " " << mod << " " << tool_version;
bool metadata_valid = !(
!url || !etag || !mod || !tool_version || pip_added == failure
);
if (metadata_valid)
{
RepoMetadata read_metadata{ url, pip_added == 1, etag, mod };
metadata_valid = (read_metadata == m_metadata)
&& (std::strcmp(tool_version, mamba_tool_version()) == 0);
}
LOG_INFO << "Metadata from SOLV are " << (metadata_valid ? "valid" : "NOT valid");
if (!metadata_valid)
{
LOG_INFO << "SOLV file was written with a previous version of "
"libsolv or mamba "
<< (tool_version != nullptr ? tool_version : "<NULL>")
<< ", updating it now!";
}
else
{
LOG_DEBUG << "Loaded from SOLV " << m_solv_file;
set_solvables_url();
repo_internalize(m_repo);
fclose(fp);
return true;
}
}
}
// fallback to JSON file
repo_empty(m_repo, /*reuseids*/ 0);
fclose(fp);
}
auto lock = LockFile(m_json_file);
#ifdef _WIN32
auto fp = _wfopen(m_json_file.wstring().c_str(), L"r");
#else
auto fp = fopen(m_json_file.string().c_str(), "r");
#endif
if (!fp)
{
throw std::runtime_error("Could not open repository file " + m_json_file.string());
}
LOG_DEBUG << "Loading JSON file '" << m_json_file.string() << "'";
int flags = Context::instance().use_only_tar_bz2 ? CONDA_ADD_USE_ONLY_TAR_BZ2 : 0;
int ret = repo_add_conda(m_repo, fp, flags);
if (ret != 0)
{
fclose(fp);
throw std::runtime_error(
"Could not read JSON repodata file (" + m_json_file.string() + ") "
+ std::string(pool_errstr(m_repo->pool))
);
}
auto lock = LockFile(json_file);
read_json(json_file);
// TODO move this to a more structured approach for repodata patching?
if (Context::instance().add_pip_as_python_dependency)
@ -451,73 +332,79 @@ namespace mamba
add_pip_as_python_dependency();
}
set_solvables_url();
repo_internalize(m_repo);
if (name() != "installed")
{
write();
write_solv(solv_file);
}
}
fclose(fp);
void MRepo::write_solv(fs::u8path filename)
{
LOG_INFO << "Writing libsolv solv file " << filename << " for repo " << name();
auto repo = srepo(*this);
repo.set_url(m_metadata.url);
repo.set_etag(m_metadata.etag);
repo.set_mod(m_metadata.mod);
repo.set_pip_added(m_metadata.pip_added);
repo.set_tool_version(MAMBA_SOLV_VERSION);
repo.internalize();
repo.write(filename);
}
void MRepo::clear(bool reuse_ids)
{
m_pool.remove_repo(id(), reuse_ids);
}
}
#include <map>
namespace mamba
{
auto MRepo::py_name() const -> std::string_view
{
return name();
}
auto MRepo::py_priority() const -> std::tuple<int, int>
{
return std::make_tuple(m_repo->priority, m_repo->subpriority);
}
auto MRepo::py_clear(bool reuse_ids) -> bool
{
clear(reuse_ids);
return true;
}
bool MRepo::write() const
auto MRepo::py_size() const -> std::size_t
{
Repodata* info;
LOG_INFO << "Writing SOLV file '" << m_solv_file.filename().string() << "'";
info = repo_add_repodata(m_repo, 0); // add new repodata for our meta info
repodata_set_str(info, SOLVID_META, REPOSITORY_TOOLVERSION, mamba_tool_version());
Id url_id = pool_str2id(m_repo->pool, "mamba:url", 1);
Id pip_added_id = pool_str2id(m_repo->pool, "mamba:pip_added", 1);
Id etag_id = pool_str2id(m_repo->pool, "mamba:etag", 1);
Id mod_id = pool_str2id(m_repo->pool, "mamba:mod", 1);
repodata_set_str(info, SOLVID_META, url_id, m_metadata.url.c_str());
repodata_set_num(info, SOLVID_META, pip_added_id, m_metadata.pip_added);
repodata_set_str(info, SOLVID_META, etag_id, m_metadata.etag.c_str());
repodata_set_str(info, SOLVID_META, mod_id, m_metadata.mod.c_str());
repodata_internalize(info);
#ifdef _WIN32
auto solv_f = _wfopen(m_solv_file.wstring().c_str(), L"wb");
#else
auto solv_f = fopen(m_solv_file.string().c_str(), "wb");
#endif
if (!solv_f)
{
LOG_ERROR << "Failed to open .solv file";
return false;
}
if (repo_write(m_repo, solv_f) != 0)
{
LOG_ERROR << "Failed to write .solv:" << pool_errstr(m_repo->pool);
fclose(solv_f);
return false;
}
if (fflush(solv_f))
{
LOG_ERROR << "Failed to flush .solv file.";
fclose(solv_f);
return false;
}
fclose(solv_f);
repodata_free(info); // delete meta info repodata again
return true;
return srepo(*this).solvable_count();
}
bool MRepo::clear(bool reuse_ids = 1)
void MRepo::py_add_extra_pkg_info(const std::map<std::string, PyExtraPkgInfo>& extra)
{
repo_free(m_repo, static_cast<int>(reuse_ids));
m_repo = nullptr;
return true;
auto repo = srepo(*this);
repo.for_each_solvable(
[&](solv::ObjSolvableView s)
{
if (auto const it = extra.find(std::string(s.name())); it != extra.cend())
{
if (auto const& noarch = it->second.noarch; !noarch.empty())
{
s.set_noarch(noarch);
}
if (auto const& repo_url = it->second.repo_url; !repo_url.empty())
{
s.set_channel(repo_url);
}
}
}
);
repo.internalize();
}
} // namespace mamba

1
libmamba/src/core/solver.cpp
View File

@ -23,6 +23,7 @@
#include "mamba/core/satisfiability_error.hpp"
#include "mamba/core/solver.hpp"
#include "mamba/core/util_string.hpp"
#include "solv-cpp/pool.hpp"
#include "solv-cpp/queue.hpp"
namespace mamba

16
libmamba/src/core/subdirdata.cpp
View File

@ -960,16 +960,18 @@ namespace mamba
return cache_dir.string();
}
expected_t<MRepo&> MSubdirData::create_repo(MPool& pool)
expected_t<MRepo> MSubdirData::create_repo(MPool& pool)
{
using return_type = expected_t<MRepo&>;
RepoMetadata meta{ m_repodata_url,
Context::instance().add_pip_as_python_dependency,
m_metadata.etag,
m_metadata.mod };
using return_type = expected_t<MRepo>;
RepoMetadata meta{
/* .url= */ rsplit(m_metadata.url, "/", 1).front(),
/* .etag= */ m_metadata.etag,
/* .mod= */ m_metadata.mod,
/* .pip_added= */ Context::instance().add_pip_as_python_dependency,
};
auto cache = cache_path();
return cache ? return_type(MRepo::create(pool, m_name, *cache, meta))
return cache ? return_type(MRepo(pool, m_name, *cache, meta))
: return_type(forward_error(cache));
}

30
libmamba/src/core/transaction.cpp
View File

@ -505,7 +505,7 @@ namespace mamba
pi_result.push_back(p);
}
MRepo& mrepo = MRepo::create(m_pool, "__explicit_specs__", pi_result);
MRepo mrepo = MRepo(m_pool, "__explicit_specs__", pi_result);
m_pool.create_whatprovides();
@ -688,11 +688,9 @@ namespace mamba
solver_get_decisionqueue(solver, decision.raw());
const Id noarch_repo_key = pool_str2id(m_pool, "solvable:noarch_type", 1);
FOR_REPO_SOLVABLES(pool_ptr->installed, p, s)
{
const char* noarch_type = solvable_lookup_str(s, noarch_repo_key);
const char* noarch_type = solvable_lookup_str(s, SOLVABLE_SOURCEARCH);
if (noarch_type == nullptr)
{
@ -791,7 +789,7 @@ namespace mamba
, m_multi_cache(caches)
{
LOG_INFO << "MTransaction::MTransaction - packages already resolved (lockfile)";
MRepo& mrepo = MRepo::create(m_pool, "__explicit_specs__", packages);
MRepo mrepo = MRepo(m_pool, "__explicit_specs__", packages);
m_pool.create_whatprovides();
solv::ObjQueue decision = {};
@ -876,8 +874,6 @@ namespace mamba
break;
}
}
m_real_repo_key = pool_str2id(m_pool, "solvable:real_repo_url", 1);
m_mrepo_key = pool_str2id(m_pool, "solvable:mrepo_url", 1);
}
// TODO rewrite this in terms of `m_transaction`
@ -1116,8 +1112,6 @@ namespace mamba
auto MTransaction::to_conda() -> to_conda_type
{
const Id real_repo_key = pool_str2id(m_pool, "solvable:real_repo_url", 1);
to_install_type to_install_structured;
to_remove_type to_remove_structured;
@ -1133,9 +1127,9 @@ namespace mamba
std::string s_json = solvable_to_json(m_pool, s).dump(4);
std::string channel;
if (solvable_lookup_str(s, real_repo_key))
if (const char* str = solvable_lookup_str(s, SOLVABLE_PACKAGER))
{
channel = solvable_lookup_str(s, real_repo_key);
channel = str;
}
else
{
@ -1212,7 +1206,9 @@ namespace mamba
for (auto& s : m_to_install)
{
std::string const s_url = raw_str_or_empty(solvable_lookup_str(s, m_mrepo_key));
std::string const s_url = raw_str_or_empty(
repo_lookup_str(s->repo, SOLVID_META, SOLVABLE_URL)
);
if (ctx.experimental && ctx.verify_artifacts)
{
@ -1539,24 +1535,22 @@ namespace mamba
const char* build_string = solvable_lookup_str(s, SOLVABLE_BUILDFLAVOR);
std::string channel;
Id real_repo_key = pool_str2id(m_pool, "solvable:real_repo_url", 1);
if (solvable_lookup_str(s, real_repo_key))
if (const char* str = solvable_lookup_str(s, SOLVABLE_PACKAGER))
{
std::string repo_key = solvable_lookup_str(s, real_repo_key);
if (repo_key == "explicit_specs")
if (std::string_view(str) == "explicit_specs")
{
channel = solvable_lookup_str(s, SOLVABLE_MEDIAFILE);
}
else
{
channel = make_channel(repo_key).canonical_name();
channel = make_channel(str).canonical_name();
}
}
else
{
// note this can and should be <unknown> when
// e.g. installing from a tarball
assert(s->repo != nullptr);
channel = s->repo->name;
assert(channel != "__explicit_specs__");
}

31
libmamba/src/solv-cpp/repo.cpp
View File

@ -19,6 +19,12 @@
#include <solv/repo.h>
#include <solv/repo_solv.h>
#include <solv/repo_write.h>
#include <solv/solvable.h>
extern "C" // Incomplete header in libsolv 0.7.23
{
#include <solv/conda.h>
#include <solv/repo_conda.h>
}
#include "mamba/core/mamba_fs.hpp"
#include "solv-cpp/repo.hpp"
@ -229,6 +235,27 @@ namespace mamba::solv
// TODO(C++20) fmt::format
auto ss = std::stringstream();
ss << "Unable to read repo solv file '" << name() << '\'';
if (const char* str = ::pool_errstr(raw()->pool))
{
ss << ", error was: " << str;
}
throw std::runtime_error(ss.str());
}
}
void ObjRepoView::legacy_read_conda_repodata(const fs::u8path& repodata_file, int flags) const
{
auto file = CFile::open(repodata_file, "rb");
const auto res = ::repo_add_conda(raw(), file.raw(), flags);
if (res != 0)
{
// TODO(C++20) fmt::format
auto ss = std::stringstream();
ss << "Unable to read repodata JSON file '" << name() << '\'';
if (const char* str = ::pool_errstr(raw()->pool))
{
ss << ", error was: " << str;
}
throw std::runtime_error(ss.str());
}
}
@ -314,12 +341,12 @@ namespace mamba::solv
auto repo_lookup_bool(const ::Repo* repo, ::Id key) -> bool
{
return ::repo_lookup_num(const_cast<::Repo*>(repo), SOLVID_META, key, 0) != 0;
return repo_lookup_num(repo, key) != 0;
}
void repo_set_bool(::Repo* repo, ::Id key, bool b)
{
::repo_set_num(repo, SOLVID_META, key, b);
repo_set_num(repo, key, b);
}
}

10
libmamba/src/solv-cpp/repo.hpp
View File

@ -223,6 +223,16 @@ namespace mamba::solv
*/
void read(const fs::u8path& solv_file) const;
/**
* Read repository information from a conda repodata.json.
*
* This function is part of libsolv and does not read all attributes of the repodata.
* It is meant to be replaced. Parsing should be done by the user and solvables
* added through the API.
* @param repodata_file A standard path with system encoding.
*/
void legacy_read_conda_repodata(const fs::u8path& repodata_file, int flags = 0) const;
/** Add an empty solvable to the repository. */
auto add_solvable() const -> std::pair<SolvableId, ObjSolvableView>;

4
libmamba/tests/src/core/test_satisfiability_error.cpp
View File

@ -144,7 +144,7 @@ namespace mamba
const auto repodata_f = create_repodata_json(tmp_dir.path, packages);
auto pool = MPool();
MRepo::create(pool, "some-name", repodata_f, "some-url");
MRepo(pool, "some-name", repodata_f, RepoMetadata{ /* .url= */ "some-url" });
auto solver = std::make_unique<MSolver>(
std::move(pool),
std::vector{ std::pair{ SOLVER_FLAG_ALLOW_DOWNGRADE, 1 } }
@ -347,7 +347,7 @@ namespace mamba
auto prefix_data = expected_value_or_throw(PrefixData::create(tmp_dir.path / "prefix"));
prefix_data.add_packages(virtual_packages);
auto pool = MPool();
auto& repo = MRepo::create(pool, prefix_data);
auto repo = MRepo(pool, prefix_data);
repo.set_installed();
auto cache = MultiPackageCache({ tmp_dir.path / "cache" });

67
libmambapy/src/main.cpp
View File

@ -163,61 +163,24 @@ PYBIND11_MODULE(bindings, m)
.def("get_tarball_path", &MultiPackageCache::get_tarball_path)
.def_property_readonly("first_writable_path", &MultiPackageCache::first_writable_path);
struct ExtraPkgInfo
{
std::string noarch;
std::string repo_url;
};
py::class_<ExtraPkgInfo>(m, "ExtraPkgInfo")
py::class_<MRepo::PyExtraPkgInfo>(m, "ExtraPkgInfo")
.def(py::init<>())
.def_readwrite("noarch", &ExtraPkgInfo::noarch)
.def_readwrite("repo_url", &ExtraPkgInfo::repo_url);
.def_readwrite("noarch", &MRepo::PyExtraPkgInfo::noarch)
.def_readwrite("repo_url", &MRepo::PyExtraPkgInfo::repo_url);
py::class_<MRepo, std::unique_ptr<MRepo, py::nodelete>>(m, "Repo")
py::class_<MRepo>(m, "Repo")
.def(py::init(
[](MPool& pool, const std::string& name, const std::string& filename, const std::string& url
) {
return std::unique_ptr<MRepo, py::nodelete>(&MRepo::create(pool, name, filename, url));
}
) { return MRepo(pool, name, filename, RepoMetadata{ /* .url=*/url }); }
))
.def(py::init([](MPool& pool, const PrefixData& data)
{ return std::unique_ptr<MRepo, py::nodelete>(&MRepo::create(pool, data)); }))
.def(
"add_extra_pkg_info",
[](const MRepo& self, const std::map<std::string, ExtraPkgInfo>& additional_info)
{
Id pkg_id;
Solvable* pkg_s;
Pool* p = self.repo()->pool;
static Id noarch_repo_key = pool_str2id(p, "solvable:noarch_type", 1);
static Id real_repo_url_key = pool_str2id(p, "solvable:real_repo_url", 1);
FOR_REPO_SOLVABLES(self.repo(), pkg_id, pkg_s)
{
std::string name = pool_id2str(p, pkg_s->name);
auto it = additional_info.find(name);
if (it != additional_info.end())
{
if (!it->second.noarch.empty())
{
solvable_set_str(pkg_s, noarch_repo_key, it->second.noarch.c_str());
}
if (!it->second.repo_url.empty())
{
solvable_set_str(pkg_s, real_repo_url_key, it->second.repo_url.c_str());
}
}
}
repo_internalize(self.repo());
}
)
.def(py::init<MPool&, const PrefixData&>())
.def("add_extra_pkg_info", &MRepo::py_add_extra_pkg_info)
.def("set_installed", &MRepo::set_installed)
.def("set_priority", &MRepo::set_priority)
.def("name", &MRepo::name)
.def("priority", &MRepo::priority)
.def("size", &MRepo::size)
.def("clear", &MRepo::clear);
.def("name", &MRepo::py_name)
.def("priority", &MRepo::py_priority)
.def("size", &MRepo::py_size)
.def("clear", &MRepo::py_clear);
py::class_<MTransaction>(m, "Transaction")
.def(py::init<>(
@ -469,12 +432,8 @@ PYBIND11_MODULE(bindings, m)
))
.def(
"create_repo",
[](MSubdirData& subdir, MPool& pool) -> MRepo&
{
auto exp_res = subdir.create_repo(pool);
return extract(exp_res);
},
py::return_value_policy::reference
[](MSubdirData& subdir, MPool& pool) -> MRepo
{ return extract(subdir.create_repo(pool)); }
)
.def("loaded", &MSubdirData::loaded)
.def(

4
micromamba/src/server.cpp
View File

@ -105,7 +105,7 @@ handle_solve_request(const microserver::Request& req, microserver::Response& res
}
prefix_data.add_packages(vpacks);
auto& installed_repo = MRepo::create(cache_entry.pool, prefix_data);
auto installed_repo = MRepo(cache_entry.pool, prefix_data);
MSolver solver(
cache_entry.pool,
@ -138,7 +138,7 @@ handle_solve_request(const microserver::Request& req, microserver::Response& res
res.send(jout.dump());
}
cache_entry.pool.remove_repo(installed_repo.id());
cache_entry.pool.remove_repo(installed_repo.id(), /* reuse_ids= */ true);
pool_set_installed(cache_entry.pool, nullptr);
}