folkslinux
/
mamba
mirror of https://github.com/mamba-org/mamba.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add multiple queries to repoquery search (#2897) 

* Remove kENUM

* Fix Hide URL confidential

* Refactor repoquery initialization

* Hide implementation headers

* Add enum_name(QueryType)

* Add QueryType_from_name

* Mulitple queries in repoquery search

* Stubgen

* No mathspec in Json query

* Remove unused ChannelContext
This commit is contained in:
Antoine Prouvost 2023-10-09 21:40:50 +02:00 committed by GitHub
parent 620ae9500c
commit 5d515fa043
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 253 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
libmamba/include/mamba/api/repoquery.hpp
View File

@ -4,6 +4,10 @@
//
// The full license is in the file LICENSE, distributed with this software.
#include <string>
#include <vector>
#include "mamba/api/configuration.hpp"
#include "mamba/core/query.hpp"
namespace mamba
@ -13,6 +17,6 @@ namespace mamba
QueryType type,
QueryResultFormat format,
bool use_local,
const std::string& query
const std::vector<std::string>& query
);
}

52
libmamba/include/mamba/core/query.hpp
View File

@ -9,17 +9,9 @@
#include <map>
#include <string>
#include <string_view>
#include <vector>
#include <solv/pool.h>
#include <solv/repo.h>
#include <solv/selection.h>
#include <solv/solver.h>
extern "C" // Incomplete header
{
#include <solv/conda.h>
}
#include "mamba/core/context.hpp"
#include "mamba/core/package_info.hpp"
#include "mamba/core/pool.hpp"
@ -47,7 +39,7 @@ namespace mamba
Query(MPool& pool);
query_result find(const std::string& query) const;
query_result find(const std::vector<std::string>& queries) const;
query_result whoneeds(const std::string& query, bool tree) const;
query_result depends(const std::string& query, bool tree) const;
@ -58,18 +50,21 @@ namespace mamba
enum class QueryType
{
kSEARCH,
kDEPENDS,
kWHONEEDS
Search,
Depends,
WhoNeeds
};
constexpr auto enum_name(QueryType t) -> std::string_view;
auto QueryType_from_name(std::string_view name) -> QueryType;
enum class QueryResultFormat
{
kJSON = 0,
kTREE = 1,
kTABLE = 2,
kPRETTY = 3,
kRECURSIVETABLE = 4,
Json = 0,
Tree = 1,
Table = 2,
Pretty = 3,
RecursiveTable = 4,
};
class query_result
@ -97,7 +92,7 @@ namespace mamba
std::ostream& table(std::ostream&) const;
std::ostream& table(std::ostream&, const std::vector<std::string_view>& fmt) const;
std::ostream& tree(std::ostream&, const GraphicsParams& graphics) const;
nlohmann::json json(ChannelContext& channel_context) const;
nlohmann::json json() const;
std::ostream& pretty(std::ostream&, const Context::OutputParams& outputParams) const;
@ -118,6 +113,25 @@ namespace mamba
package_id_list m_pkg_id_list = {};
ordered_package_list m_ordered_pkg_id_list = {};
};
/********************
* Implementation *
********************/
constexpr auto enum_name(QueryType t) -> std::string_view
{
switch (t)
{
case QueryType::Search:
return "Search";
case QueryType::WhoNeeds:
return "WhoNeeds";
case QueryType::Depends:
return "Depends";
}
throw std::invalid_argument("Invalid enum value");
}
} // namespace mamba
#endif // MAMBA_QUERY_HPP

174
libmamba/src/api/repoquery.cpp
View File

@ -17,76 +17,90 @@
namespace mamba
{
namespace
{
auto repoquery_init(Configuration& config, QueryResultFormat format, bool use_local)
{
auto& ctx = config.context();
config.at("use_target_prefix_fallback").set_value(true);
config.at("target_prefix_checks")
.set_value(MAMBA_ALLOW_EXISTING_PREFIX | MAMBA_ALLOW_MISSING_PREFIX);
config.load();
ChannelContext channel_context{ ctx };
MPool pool{ channel_context };
// bool installed = (type == QueryType::kDepends) || (type == QueryType::kWhoneeds);
MultiPackageCache package_caches(ctx.pkgs_dirs, ctx.validation_params);
if (use_local)
{
if (format != QueryResultFormat::Json)
{
Console::stream() << "Using local repodata..." << std::endl;
}
auto exp_prefix_data = PrefixData::create(
ctx.prefix_params.target_prefix,
channel_context
);
if (!exp_prefix_data)
{
// TODO: propagate tl::expected mechanism
throw std::runtime_error(exp_prefix_data.error().what());
}
PrefixData& prefix_data = exp_prefix_data.value();
MRepo(pool, prefix_data);
if (format != QueryResultFormat::Json)
{
Console::stream()
<< "Loaded current active prefix: " << ctx.prefix_params.target_prefix
<< std::endl;
}
}
else
{
if (format != QueryResultFormat::Json)
{
Console::stream() << "Getting repodata from channels..." << std::endl;
}
auto exp_load = load_channels(pool, package_caches, 0);
if (!exp_load)
{
throw std::runtime_error(exp_load.error().what());
}
}
return pool;
}
}
void repoquery(
Configuration& config,
QueryType type,
QueryResultFormat format,
bool use_local,
const std::string& query
const std::vector<std::string>& queries
)
{
auto& ctx = config.context();
config.at("use_target_prefix_fallback").set_value(true);
config.at("target_prefix_checks")
.set_value(MAMBA_ALLOW_EXISTING_PREFIX | MAMBA_ALLOW_MISSING_PREFIX);
config.load();
ChannelContext channel_context{ ctx };
MPool pool{ channel_context };
// bool installed = (type == QueryType::kDepends) || (type == QueryType::kWhoneeds);
MultiPackageCache package_caches(ctx.pkgs_dirs, ctx.validation_params);
if (use_local)
{
if (format != QueryResultFormat::kJSON)
{
Console::stream() << "Using local repodata..." << std::endl;
}
auto exp_prefix_data = PrefixData::create(ctx.prefix_params.target_prefix, channel_context);
if (!exp_prefix_data)
{
// TODO: propagate tl::expected mechanism
throw std::runtime_error(exp_prefix_data.error().what());
}
PrefixData& prefix_data = exp_prefix_data.value();
MRepo(pool, prefix_data);
if (format != QueryResultFormat::kJSON)
{
Console::stream() << "Loaded current active prefix: "
<< ctx.prefix_params.target_prefix << std::endl;
}
}
else
{
if (format != QueryResultFormat::kJSON)
{
Console::stream() << "Getting repodata from channels..." << std::endl;
}
auto exp_load = load_channels(pool, package_caches, 0);
if (!exp_load)
{
throw std::runtime_error(exp_load.error().what());
}
}
auto pool = repoquery_init(config, format, use_local);
Query q(pool);
if (type == QueryType::kSEARCH)
if (type == QueryType::Search)
{
if (ctx.output_params.json)
{
std::cout << q.find(query).groupby("name").json(pool.channel_context()).dump(4);
std::cout << q.find(queries).groupby("name").json().dump(4);
}
else
{
std::cout << "\n" << std::endl;
auto res = q.find(query);
auto res = q.find(queries);
switch (format)
{
case QueryResultFormat::kJSON:
std::cout << res.json(pool.channel_context()).dump(4);
case QueryResultFormat::Json:
std::cout << res.json().dump(4);
break;
case QueryResultFormat::kPRETTY:
case QueryResultFormat::Pretty:
res.pretty(std::cout, ctx.output_params);
break;
default:
@ -99,49 +113,57 @@ namespace mamba
}
}
}
else if (type == QueryType::kDEPENDS)
else if (type == QueryType::Depends)
{
if (queries.size() != 1)
{
throw std::invalid_argument("Only one query supported for 'depends'.");
}
auto res = q.depends(
query,
format == QueryResultFormat::kTREE || format == QueryResultFormat::kRECURSIVETABLE
queries.front(),
format == QueryResultFormat::Tree || format == QueryResultFormat::RecursiveTable
);
switch (format)
{
case QueryResultFormat::kTREE:
case QueryResultFormat::kPRETTY:
case QueryResultFormat::Tree:
case QueryResultFormat::Pretty:
res.tree(std::cout, config.context().graphics_params);
break;
case QueryResultFormat::kJSON:
std::cout << res.json(pool.channel_context()).dump(4);
case QueryResultFormat::Json:
std::cout << res.json().dump(4);
break;
case QueryResultFormat::kTABLE:
case QueryResultFormat::kRECURSIVETABLE:
case QueryResultFormat::Table:
case QueryResultFormat::RecursiveTable:
res.sort("name").table(std::cout);
}
if (res.empty() && format != QueryResultFormat::kJSON)
if (res.empty() && format != QueryResultFormat::Json)
{
std::cout << query
<< " may not be installed. Try giving a channel with '-c,--channel' option for remote repoquery"
std::cout << queries.front(
) << " may not be installed. Try giving a channel with '-c,--channel' option for remote repoquery"
<< std::endl;
}
}
else if (type == QueryType::kWHONEEDS)
else if (type == QueryType::WhoNeeds)
{
if (queries.size() != 1)
{
throw std::invalid_argument("Only one query supported for 'whoneeds'.");
}
auto res = q.whoneeds(
query,
format == QueryResultFormat::kTREE || format == QueryResultFormat::kRECURSIVETABLE
queries.front(),
format == QueryResultFormat::Tree || format == QueryResultFormat::RecursiveTable
);
switch (format)
{
case QueryResultFormat::kTREE:
case QueryResultFormat::kPRETTY:
case QueryResultFormat::Tree:
case QueryResultFormat::Pretty:
res.tree(std::cout, config.context().graphics_params);
break;
case QueryResultFormat::kJSON:
std::cout << res.json(pool.channel_context()).dump(4);
case QueryResultFormat::Json:
std::cout << res.json().dump(4);
break;
case QueryResultFormat::kTABLE:
case QueryResultFormat::kRECURSIVETABLE:
case QueryResultFormat::Table:
case QueryResultFormat::RecursiveTable:
res.sort("name").table(
std::cout,
{ "Name",
@ -149,15 +171,15 @@ namespace mamba
"Build",
printers::alignmentMarker(printers::alignment::left),
printers::alignmentMarker(printers::alignment::right),
util::concat("Depends:", query),
util::concat("Depends:", queries.front()),
"Channel",
"Subdir" }
);
}
if (res.empty() && format != QueryResultFormat::kJSON)
if (res.empty() && format != QueryResultFormat::Json)
{
std::cout << query
<< " may not be installed. Try giving a channel with '-c,--channel' option for remote repoquery"
std::cout << queries.front(
) << " may not be installed. Try giving a channel with '-c,--channel' option for remote repoquery"
<< std::endl;
}
}

65
libmamba/src/core/query.cpp
View File

@ -14,10 +14,17 @@
#include <fmt/format.h>
#include <fmt/ostream.h>
#include <solv/evr.h>
#include <solv/pool.h>
#include <solv/repo.h>
#include <solv/selection.h>
#include <solv/solver.h>
#include <spdlog/spdlog.h>
extern "C" // Incomplete header
{
#include <solv/conda.h>
}
#include "mamba/core/context.hpp"
#include "mamba/core/match_spec.hpp"
#include "mamba/core/output.hpp"
#include "mamba/core/package_info.hpp"
#include "mamba/core/query.hpp"
@ -347,16 +354,19 @@ namespace mamba
}
}
query_result Query::find(const std::string& query) const
query_result Query::find(const std::vector<std::string>& queries) const
{
solv::ObjQueue job, solvables;
const Id id = pool_conda_matchspec(m_pool.get(), query.c_str());
if (!id)
for (const auto& query : queries)
{
throw std::runtime_error("Could not generate query for " + query);
const Id id = pool_conda_matchspec(m_pool.get(), query.c_str());
if (!id)
{
throw std::runtime_error("Could not generate query for " + query);
}
job.push_back(SOLVER_SOLVABLE_PROVIDES, id);
}
job.push_back(SOLVER_SOLVABLE_PROVIDES, id);
selection_solvables(m_pool.get(), job.raw(), solvables.raw());
query_result::dependency_graph g;
@ -382,7 +392,11 @@ namespace mamba
g.add_node(std::move(pkg_info).value());
}
return query_result(QueryType::kSEARCH, query, std::move(g));
return query_result(
QueryType::Search,
fmt::format("{}", fmt::join(queries, " ")), // Yes this is disgusting
std::move(g)
);
}
query_result Query::whoneeds(const std::string& query, bool tree) const
@ -421,7 +435,7 @@ namespace mamba
g.add_node(std::move(pkg_info).value());
}
}
return query_result(QueryType::kWHONEEDS, query, std::move(g));
return query_result(QueryType::WhoNeeds, query, std::move(g));
}
query_result Query::depends(const std::string& query, bool tree) const
@ -466,7 +480,29 @@ namespace mamba
walk_graph(m_pool, g, node_id, latest, visited, not_found, depth);
}
return query_result(QueryType::kDEPENDS, query, std::move(g));
return query_result(QueryType::Depends, query, std::move(g));
}
/******************************
* QueryType implementation *
******************************/
auto QueryType_from_name(std::string_view name) -> QueryType
{
auto l_name = util::to_lower(name);
if (l_name == "search")
{
return QueryType::Search;
}
if (l_name == "depends")
{
return QueryType::Depends;
}
if (l_name == "whoneeds")
{
return QueryType::WhoNeeds;
}
throw std::invalid_argument(fmt::format("Invalid enum name \"{}\"", name));
}
/*******************************
@ -831,14 +867,11 @@ namespace mamba
return out;
}
nlohmann::json query_result::json(ChannelContext& channel_context) const
nlohmann::json query_result::json() const
{
nlohmann::json j;
std::string query_type = m_type == QueryType::kSEARCH
? "search"
: (m_type == QueryType::kDEPENDS ? "depends" : "whoneeds");
j["query"] = { { "query", MatchSpec{ m_query, channel_context }.conda_build_form() },
{ "type", query_type } };
std::string query_type = std::string(util::to_lower(enum_name(m_type)));
j["query"] = { { "query", m_query }, { "type", query_type } };
std::string msg = m_pkg_id_list.empty() ? "No entries matching \"" + m_query + "\" found"
: "";
@ -857,7 +890,7 @@ namespace mamba
j["result"]["pkgs"].push_back(std::move(pkg_info_json));
}
if (m_type != QueryType::kSEARCH && !m_pkg_id_list.empty())
if (m_type != QueryType::Search && !m_pkg_id_list.empty())
{
j["result"]["graph_roots"] = nlohmann::json::array();
if (!m_dep_graph.successors(0).empty())

4
libmamba/src/specs/conda_url.cpp
View File

@ -328,7 +328,9 @@ namespace mamba::specs
(strip_scheme == StripScheme::no) ? "://" : "",
user(Decode::yes),
password(Decode::no).empty() ? "" : ":",
(hide_confifential == HideConfidential::no) ? password(Decode::yes) : "*****",
password(Decode::no).empty()
? ""
: ((hide_confifential == HideConfidential::no) ? password(Decode::yes) : "*****"),
user(Decode::no).empty() ? "" : "@",
host(Decode::yes),
port().empty() ? "" : ":",

4
libmamba/src/util/url.cpp
View File

@ -509,7 +509,9 @@ namespace mamba::util
(strip_scheme == StripScheme::no) ? "://" : "",
user(Decode::yes),
m_password.empty() ? "" : ":",
(hide_confidential == HideConfidential::no) ? password(Decode::yes) : "*****",
password(Decode::no).empty()
? ""
: ((hide_confidential == HideConfidential::no) ? password(Decode::yes) : "*****"),
m_user.empty() ? "" : "@",
host(Decode::yes),
m_port.empty() ? "" : ":",

3
libmambapy/libmambapy/__init__.pyi
View File

@ -1235,7 +1235,10 @@ class ProblemsGraph:
class Query:
def __init__(self, arg0: Pool) -> None: ...
def depends(self, arg0: str, arg1: QueryFormat) -> str: ...
@typing.overload
def find(self, arg0: str, arg1: QueryFormat) -> str: ...
@typing.overload
def find(self, arg0: typing.List[str], arg1: QueryFormat) -> str: ...
def whoneeds(self, arg0: str, arg1: QueryFormat) -> str: ...
pass

62
libmambapy/src/main.cpp
View File

@ -510,39 +510,41 @@ PYBIND11_MODULE(bindings, m)
.value("PRETTY", query::RESULT_FORMAT::PRETTY)
.value("RECURSIVETABLE", query::RESULT_FORMAT::RECURSIVETABLE);
auto queries_find = [](const Query& q,
const std::vector<std::string>& queries,
const query::RESULT_FORMAT format) -> std::string
{
query_result res = q.find(queries);
std::stringstream res_stream;
switch (format)
{
case query::JSON:
res_stream << res.groupby("name").json().dump(4);
break;
case query::TREE:
case query::TABLE:
case query::RECURSIVETABLE:
res.groupby("name").table(res_stream);
break;
case query::PRETTY:
res.groupby("name").pretty(res_stream, mambapy::singletons.context().output_params);
}
if (res.empty() && format != query::JSON)
{
res_stream << fmt::format("{}", fmt::join(queries, " "))
<< " may not be installed. Try specifying a channel with '-c,--channel' option\n";
}
return res_stream.str();
};
py::class_<Query>(m, "Query")
.def(py::init<MPool&>())
.def(
"find",
[](const Query& q, const std::string& query, const query::RESULT_FORMAT format) -> std::string
{
query_result res = q.find(query);
std::stringstream res_stream;
switch (format)
{
case query::JSON:
res_stream
<< res.groupby("name").json(mambapy::singletons.channel_context()).dump(4);
break;
case query::TREE:
case query::TABLE:
case query::RECURSIVETABLE:
res.groupby("name").table(res_stream);
break;
case query::PRETTY:
res.groupby("name").pretty(
res_stream,
mambapy::singletons.context().output_params
);
}
if (res.empty() && format != query::JSON)
{
res_stream << query
<< " may not be installed. Try specifying a channel with '-c,--channel' option\n";
}
return res_stream.str();
}
[queries_find](const Query& q, const std::string& query, const query::RESULT_FORMAT format)
-> std::string { return queries_find(q, { query }, format); }
)
.def("find", queries_find)
.def(
"whoneeds",
[](const Query& q, const std::string& query, const query::RESULT_FORMAT format) -> std::string
@ -557,7 +559,7 @@ PYBIND11_MODULE(bindings, m)
res.tree(res_stream, mambapy::singletons.context().graphics_params);
break;
case query::JSON:
res_stream << res.json(mambapy::singletons.channel_context()).dump(4);
res_stream << res.json().dump(4);
break;
case query::TABLE:
case query::RECURSIVETABLE:
@ -597,7 +599,7 @@ PYBIND11_MODULE(bindings, m)
res.tree(res_stream, mambapy::singletons.context().graphics_params);
break;
case query::JSON:
res_stream << res.json(mambapy::singletons.channel_context()).dump(4);
res_stream << res.json().dump(4);
break;
case query::TABLE:
case query::RECURSIVETABLE:

60
micromamba/src/repoquery.cpp
View File

@ -9,24 +9,6 @@
#include "common_options.hpp"
mamba::QueryType
str_to_qtype(const std::string& s)
{
if (s == "search")
{
return mamba::QueryType::kSEARCH;
}
if (s == "depends")
{
return mamba::QueryType::kDEPENDS;
}
if (s == "whoneeds")
{
return mamba::QueryType::kWHONEEDS;
}
throw std::runtime_error("Could not parse query type");
}
void
set_common_search(CLI::App* subcom, mamba::Configuration& config, bool is_repoquery)
{
@ -75,55 +57,59 @@ set_common_search(CLI::App* subcom, mamba::Configuration& config, bool is_repoqu
auto& platform = config.at("platform");
subcom->add_option("--platform", platform.get_cli_config<std::string>(), platform.description());
auto specs_has_wildcard = [](auto first, auto last) -> bool
{
auto has_wildcard = [](std::string_view spec) -> bool
{ return spec.find('*') != std::string_view::npos; };
return std::any_of(first, last, has_wildcard);
};
subcom->callback(
[&]
{
using namespace mamba;
auto qtype = str_to_qtype(query_type);
QueryResultFormat format = QueryResultFormat::kTABLE;
auto qtype = QueryType_from_name(query_type);
QueryResultFormat format = QueryResultFormat::Table;
bool use_local = true;
switch (qtype)
{
case QueryType::kSEARCH:
format = QueryResultFormat::kTABLE;
case QueryType::Search:
format = QueryResultFormat::Table;
use_local = local > 0; // use remote repodata by default for `search`
break;
case QueryType::kDEPENDS:
format = QueryResultFormat::kTABLE;
use_local = (local == -1) || (local > 0);
break;
case QueryType::kWHONEEDS:
format = QueryResultFormat::kTABLE;
case QueryType::Depends:
case QueryType::WhoNeeds:
format = QueryResultFormat::Table;
use_local = (local == -1) || (local > 0);
break;
}
if (qtype == QueryType::kDEPENDS && recursive)
if (qtype == QueryType::Depends && recursive)
{
format = QueryResultFormat::kRECURSIVETABLE;
format = QueryResultFormat::RecursiveTable;
}
if (qtype == QueryType::kDEPENDS && show_as_tree)
if (qtype == QueryType::Depends && show_as_tree)
{
format = QueryResultFormat::kTREE;
format = QueryResultFormat::Tree;
}
// Best guess to detect wildcard search; if there's no wildcard search, we want to show
// the pretty single package view.
if (qtype == QueryType::kSEARCH
&& (pretty_print || specs[0].find("*") == std::string::npos))
if (qtype == QueryType::Search
&& (pretty_print || specs_has_wildcard(specs.cbegin(), specs.cend())))
{
format = QueryResultFormat::kPRETTY;
format = QueryResultFormat::Pretty;
}
if (config.at("json").compute().value<bool>())
{
format = QueryResultFormat::kJSON;
format = QueryResultFormat::Json;
}
auto& channels = config.at("channels").compute().value<std::vector<std::string>>();
use_local = use_local && channels.empty();
repoquery(config, qtype, format, use_local, specs[0]);
repoquery(config, qtype, format, use_local, specs);
}
);
}

8
micromamba/tests/test_repoquery.py
View File

@ -22,7 +22,7 @@ def yaml_env(tmp_prefix: Path) -> None:
def test_depends(yaml_env: Path):
res = helpers.umamba_repoquery("depends", "yaml=0.2.5", "--json")
assert res["query"]["query"] == "yaml =0.2.5*"
assert res["query"]["query"] == "yaml=0.2.5"
assert res["query"]["type"] == "depends"
pkgs = res["result"]["pkgs"]
@ -67,7 +67,7 @@ def test_depends_not_installed_with_channel(yaml_env: Path, with_platform):
"depends", "-c", "conda-forge", "xtensor=0.24.5", "--json"
)
assert res["query"]["query"] == "xtensor =0.24.5*"
assert res["query"]["query"] == "xtensor=0.24.5"
assert res["query"]["type"] == "depends"
assert "conda-forge" in res["result"]["graph_roots"][0]["channel"]
assert res["result"]["graph_roots"][0]["name"] == "xtensor"
@ -155,7 +155,7 @@ def test_whoneeds_not_installed_with_channel(yaml_env: Path, with_platform):
"whoneeds", "-c", "conda-forge", "xtensor=0.24.5", "--json"
)
assert res["query"]["query"] == "xtensor =0.24.5*"
assert res["query"]["query"] == "xtensor=0.24.5"
assert res["query"]["type"] == "whoneeds"
pkgs = res["result"]["pkgs"]
@ -233,7 +233,7 @@ def test_remote_search_not_installed_pkg(yaml_env: Path):
"search", "-c", "conda-forge", "xtensor=0.24.5", "--json"
)
assert res["query"]["query"] == "xtensor =0.24.5*"
assert res["query"]["query"] == "xtensor=0.24.5"
assert res["query"]["type"] == "search"
assert "conda-forge" in res["result"]["pkgs"][0]["channel"]
assert res["result"]["pkgs"][0]["name"] == "xtensor"