add some utility functions for better error reporting and refactor with MQueue (#1789)

libmamba/CMakeLists.txt

@@ -207,6 +207,7 @@ set(LIBMAMBA_HEADERS
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/progress_bar.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/pinning.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/query.hpp
+    ${LIBMAMBA_INCLUDE_DIR}/mamba/core/queue.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/repo.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/shell_init.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/solver.hpp

libmamba/include/mamba/core/pool.hpp

@@ -36,6 +36,9 @@ namespace mamba
         void set_debuglevel();
         void create_whatprovides();
 
+        std::vector<Id> select_solvables(Id id);
+        Id matchspec2id(const std::string& ms);
+
         operator Pool*();
 
         MRepo& add_repo(MRepo&& repo);

libmamba/include/mamba/core/queue.hpp

@@ -0,0 +1,102 @@
+// Copyright (c) 2019, QuantStack and Mamba Contributors
+//
+// Distributed under the terms of the BSD 3-Clause License.
+//
+// The full license is in the file LICENSE, distributed with this software.
+
+#ifndef MAMBA_CORE_QUEUE_HPP
+#define MAMBA_CORE_QUEUE_HPP
+
+#include <vector>
+
+extern "C"
+{
+#include "solv/queue.h"
+}
+
+namespace mamba
+{
+    class MQueue
+    {
+    public:
+        MQueue()
+            : p_queue(new Queue)
+        {
+            queue_init(p_queue);
+
+            if (!p_queue)
+                throw std::runtime_error("libsolv error: could not initialize Queue");
+        }
+
+        ~MQueue()
+        {
+            queue_free(p_queue);
+        }
+
+        void push(Id id)
+        {
+            queue_push(p_queue, id);
+        }
+
+        void push(Id id1, Id id2)
+        {
+            queue_push2(p_queue, id1, id2);
+        }
+
+        int count()
+        {
+            return p_queue->count;
+        }
+
+        void clear()
+        {
+            queue_empty(p_queue);
+        }
+
+        Id& operator[](int idx)
+        {
+            return p_queue->elements[idx];
+        }
+
+        const Id& operator[](int idx) const
+        {
+            return p_queue->elements[idx];
+        }
+
+        Id* begin()
+        {
+            return &p_queue->elements[0];
+        }
+
+        Id* end()
+        {
+            return &p_queue->elements[p_queue->count];
+        }
+
+        const Id* begin() const
+        {
+            return &p_queue->elements[0];
+        }
+
+        const Id* end() const
+        {
+            return &p_queue->elements[p_queue->count];
+        }
+
+        operator Queue*()
+        {
+            return p_queue;
+        }
+
+        template <template <typename, typename...> class C>
+        C<Id> as()
+        {
+            return C<Id>(begin(), end());
+        }
+
+    private:
+        Queue* p_queue;
+    };
+}  // namespace mamba
+
+#endif  // MAMBA_POOL_HPP

libmamba/src/core/pool.cpp

@@ -7,6 +7,8 @@
 extern "C"
 {
 #include <solv/pool.h>
+#include <solv/solver.h>
+#include <solv/selection.h>
 }
 
 #include "spdlog/spdlog.h"
@@ -14,6 +16,7 @@ extern "C"
 #include "mamba/core/context.hpp"
 #include "mamba/core/pool.hpp"
 #include "mamba/core/output.hpp"
+#include "mamba/core/queue.hpp"
 
 namespace mamba
 {
@@ -79,6 +82,22 @@ namespace mamba
         return m_pool;
     }
 
+    std::vector<Id> MPool::select_solvables(Id matchspec)
+    {
+        MQueue job, solvables;
+        job.push(SOLVER_SOLVABLE_PROVIDES, matchspec);
+        selection_solvables(m_pool, job, solvables);
+        return solvables.as<std::vector>();
+    }
+
+    Id MPool::matchspec2id(const std::string& ms)
+    {
+        Id id = pool_conda_matchspec(m_pool, ms.c_str());
+        if (!id)
+            throw std::runtime_error("libsolv error: could not create matchspec from string");
+        return id;
+    }
+
     MRepo& MPool::add_repo(MRepo&& repo)
     {
         m_repo_list.push_back(std::move(repo));

libmamba/src/core/prefix_data.cpp

@@ -13,6 +13,7 @@ extern "C"
 #include "mamba/core/output.hpp"
 
 #include "mamba/core/pool.hpp"
+#include "mamba/core/queue.hpp"
 #include "mamba/core/repo.hpp"
 
 namespace mamba
@@ -78,9 +79,7 @@ namespace mamba
         std::vector<PackageInfo> result;
         MPool pool;
 
-        Queue q;
-        queue_init(&q);
-
+        MQueue q;
         {
             // TODO check prereq marker to `pip` if it's part of the installed packages
             // so that it gets installed after Python.
@@ -92,14 +91,14 @@ namespace mamba
 
             FOR_REPO_SOLVABLES(repo.repo(), pkg_id, s)
             {
-                queue_push(&q, pkg_id);
+                q.push(pkg_id);
             }
         }
 
         Pool* pp = pool;
         pp->installed = nullptr;
 
-        Transaction* t = transaction_create_decisionq(pool, &q, nullptr);
+        Transaction* t = transaction_create_decisionq(pool, q, nullptr);
         transaction_order(t, 0);
 
         for (int i = 0; i < t->steps.count; i++)
@@ -123,7 +122,6 @@ namespace mamba
                         "Package not found in prefix records or other unexpected condition");
             }
         }
-        queue_free(&q);
         return result;
     }
 

libmamba/src/core/query.cpp

@@ -18,6 +18,7 @@ extern "C"
 #include "mamba/core/match_spec.hpp"
 #include "mamba/core/output.hpp"
 #include "mamba/core/package_info.hpp"
+#include "mamba/core/queue.hpp"
 #include "mamba/core/util.hpp"
 #include "mamba/core/url.hpp"
 
@@ -45,19 +46,17 @@ namespace mamba
 
             while (req != 0)
             {
-                Queue job, rec_solvables;
-                queue_init(&rec_solvables);
-                queue_init(&job);
+                MQueue job, rec_solvables;
                 // the following prints the requested version
-                queue_push2(&job, SOLVER_SOLVABLE_PROVIDES, req);
-                selection_solvables(pool, &job, &rec_solvables);
+                job.push(SOLVER_SOLVABLE_PROVIDES, req);
+                selection_solvables(pool, job, rec_solvables);
 
-                if (rec_solvables.count != 0)
+                if (rec_solvables.count() != 0)
                 {
                     Solvable* rs = nullptr;
-                    for (int i = 0; i < rec_solvables.count; i++)
+                    for (auto& el : rec_solvables)
                     {
-                        rs = pool_id2solvable(pool, rec_solvables.elements[i]);
+                        rs = pool_id2solvable(pool, el);
                         if (rs->name == req)
                         {
                             break;
@@ -92,7 +91,6 @@ namespace mamba
                         dep_graph.add_edge(parent, it->second);
                     }
                 }
-                queue_free(&rec_solvables);
                 ++reqp;
                 req = *reqp;
             }
@@ -108,17 +106,16 @@ namespace mamba
         {
             auto* pool = s->repo->pool;
             // figure out who requires `s`
-            Queue solvables;
-            queue_init(&solvables);
+            MQueue solvables;
 
-            pool_whatmatchesdep(pool, SOLVABLE_REQUIRES, s->name, &solvables, -1);
+            pool_whatmatchesdep(pool, SOLVABLE_REQUIRES, s->name, solvables, -1);
 
-            if (solvables.count != 0)
+            if (solvables.count() != 0)
             {
                 Solvable* rs;
-                for (int i = 0; i < solvables.count; i++)
+                for (auto& el : solvables)
                 {
-                    rs = pool_id2solvable(pool, solvables.elements[i]);
+                    rs = pool_id2solvable(pool, el);
                     auto it = visited.find(rs);
                     if (it == visited.end())
                     {
@@ -132,7 +129,6 @@ namespace mamba
                         dep_graph.add_edge(parent, it->second);
                     }
                 }
-                queue_free(&solvables);
             }
         }
     }
@@ -202,26 +198,24 @@ namespace mamba
 
     query_result Query::find(const std::string& query) const
     {
-        Queue job, solvables;
-        queue_init(&job);
-        queue_init(&solvables);
+        MQueue job, solvables;
 
         Id id = pool_conda_matchspec(m_pool.get(), query.c_str());
         if (id)
         {
-            queue_push2(&job, SOLVER_SOLVABLE_PROVIDES, id);
+            job.push(SOLVER_SOLVABLE_PROVIDES, id);
         }
         else
         {
             throw std::runtime_error("Could not generate query for " + query);
         }
 
-        selection_solvables(m_pool.get(), &job, &solvables);
+        selection_solvables(m_pool.get(), job, solvables);
         query_result::dependency_graph g;
 
         Pool* pool = m_pool.get();
-        std::sort(solvables.elements,
-                  solvables.elements + solvables.count,
+        std::sort(solvables.begin(),
+                  solvables.end(),
                   [pool](Id a, Id b)
                   {
                       Solvable* sa;
@@ -231,28 +225,23 @@ namespace mamba
                       return (pool_evrcmp(pool, sa->evr, sb->evr, EVRCMP_COMPARE) > 0);
                   });
 
-        for (int i = 0; i < solvables.count; i++)
+        for (auto& el : solvables)
         {
-            Solvable* s = pool_id2solvable(m_pool.get(), solvables.elements[i]);
+            Solvable* s = pool_id2solvable(m_pool.get(), el);
             g.add_node(PackageInfo(s));
         }
 
-        queue_free(&job);
-        queue_free(&solvables);
-
         return query_result(QueryType::kSEARCH, query, std::move(g));
     }
 
     query_result Query::whoneeds(const std::string& query, bool tree) const
     {
-        Queue job, solvables;
-        queue_init(&job);
-        queue_init(&solvables);
+        MQueue job, solvables;
 
         Id id = pool_conda_matchspec(m_pool.get(), query.c_str());
         if (id)
         {
-            queue_push2(&job, SOLVER_SOLVABLE_PROVIDES, id);
+            job.push(SOLVER_SOLVABLE_PROVIDES, id);
         }
         else
         {
@@ -263,10 +252,10 @@ namespace mamba
 
         if (tree)
         {
-            selection_solvables(m_pool.get(), &job, &solvables);
-            if (solvables.count > 0)
+            selection_solvables(m_pool.get(), job, solvables);
+            if (solvables.count() > 0)
             {
-                Solvable* latest = pool_id2solvable(m_pool.get(), solvables.elements[0]);
+                Solvable* latest = pool_id2solvable(m_pool.get(), solvables[0]);
                 const auto node_id = g.add_node(PackageInfo(latest));
                 std::map<Solvable*, size_t> visited = { { latest, node_id } };
                 reverse_walk_graph(g, node_id, latest, visited);
@@ -274,10 +263,10 @@ namespace mamba
         }
         else
         {
-            pool_whatmatchesdep(m_pool.get(), SOLVABLE_REQUIRES, id, &solvables, -1);
-            for (int i = 0; i < solvables.count; i++)
+            pool_whatmatchesdep(m_pool.get(), SOLVABLE_REQUIRES, id, solvables, -1);
+            for (auto& el : solvables)
             {
-                Solvable* s = pool_id2solvable(m_pool.get(), solvables.elements[i]);
+                Solvable* s = pool_id2solvable(m_pool.get(), el);
                 g.add_node(PackageInfo(s));
             }
         }
@@ -286,14 +275,12 @@ namespace mamba
 
     query_result Query::depends(const std::string& query, bool tree) const
     {
-        Queue job, solvables;
-        queue_init(&job);
-        queue_init(&solvables);
+        MQueue job, solvables;
 
         Id id = pool_conda_matchspec(m_pool.get(), query.c_str());
         if (id)
         {
-            queue_push2(&job, SOLVER_SOLVABLE_PROVIDES, id);
+            job.push(SOLVER_SOLVABLE_PROVIDES, id);
         }
         else
         {
@@ -301,16 +288,16 @@ namespace mamba
         }
 
         query_result::dependency_graph g;
-        selection_solvables(m_pool.get(), &job, &solvables);
+        selection_solvables(m_pool.get(), job, solvables);
 
         int depth = tree ? -1 : 1;
 
-        auto find_latest = [&](Queue& solvables) -> Solvable*
+        auto find_latest = [&](MQueue& solvables) -> Solvable*
         {
-            Solvable* latest = pool_id2solvable(m_pool.get(), solvables.elements[0]);
-            for (int i = 1; i < solvables.count; ++i)
+            Solvable* latest = pool_id2solvable(m_pool.get(), solvables[0]);
+            for (int i = 1; i < solvables.count(); ++i)
             {
-                Solvable* s = pool_id2solvable(m_pool.get(), solvables.elements[i]);
+                Solvable* s = pool_id2solvable(m_pool.get(), solvables[i]);
                 if (pool_evrcmp(m_pool.get(), s->evr, latest->evr, 0) > 0)
                 {
                     latest = s;
@@ -319,7 +306,7 @@ namespace mamba
             return latest;
         };
 
-        if (solvables.count > 0)
+        if (solvables.count() != 0)
         {
             Solvable* latest = find_latest(solvables);
             const auto node_id = g.add_node(PackageInfo(latest));
@@ -328,9 +315,6 @@ namespace mamba
             walk_graph(g, node_id, latest, visited, not_found, depth);
         }
 
-        queue_free(&job);
-        queue_free(&solvables);
-
         return query_result(QueryType::kDEPENDS, query, std::move(g));
     }
 

libmamba/src/core/transaction.cpp

@@ -14,6 +14,7 @@
 #include "mamba/core/match_spec.hpp"
 #include "mamba/core/output.hpp"
 #include "mamba/core/pool.hpp"
+#include "mamba/core/queue.hpp"
 #include "mamba/core/thread_utils.hpp"
 #include "mamba/core/execution.hpp"
 #include "mamba/core/util_scope.hpp"
@@ -474,31 +475,29 @@ namespace mamba
         pool.create_whatprovides();
 
         // Just add the packages we want to remove directly to the transaction
-        Queue q, job, decision;
-        queue_init(&q);
-        queue_init(&job);
-        queue_init(&decision);
+        MQueue q, job, decision;
 
         std::vector<std::string> not_found;
         for (auto& s : specs_to_remove)
         {
-            queue_empty(&job);
-            queue_empty(&q);
+            job.clear();
+            q.clear();
+
             Id id = pool_conda_matchspec((Pool*) pool, s.conda_build_form().c_str());
             if (id)
             {
-                queue_push2(&job, SOLVER_SOLVABLE_PROVIDES, id);
+                job.push(SOLVER_SOLVABLE_PROVIDES, id);
             }
-            selection_solvables((Pool*) pool, &job, &q);
+            selection_solvables(pool, job, q);
 
-            if (q.count == 0)
+            if (q.count() == 0)
             {
                 not_found.push_back("\n - " + s.str());
             }
-            for (std::size_t i = 0; i < size_t(q.count); i++)
+            for (auto& el : q)
             {
                 // To remove, these have to be negative
-                queue_push(&decision, -q.elements[i]);
+                decision.push(-el);
             }
         }
 
@@ -508,8 +507,8 @@ namespace mamba
             throw std::runtime_error("Could not find packages to remove:" + join("", not_found));
         }
 
-        selection_solvables((Pool*) pool, &job, &q);
-        bool remove_success = size_t(q.count) >= specs_to_remove.size();
+        selection_solvables(pool, job, q);
+        bool remove_success = size_t(q.count()) >= specs_to_remove.size();
         Console::instance().json_write({ { "success", remove_success } });
         Id pkg_id;
         Solvable* solvable;
@@ -517,12 +516,10 @@ namespace mamba
         // find repo __explicit_specs__ and install all packages from it
         FOR_REPO_SOLVABLES(mrepo.repo(), pkg_id, solvable)
         {
-            queue_push(&decision, pkg_id);
+            decision.push(pkg_id);
         }
 
-        queue_free(&job);
-
-        m_transaction = transaction_create_decisionq((Pool*) pool, &decision, nullptr);
+        m_transaction = transaction_create_decisionq(pool, decision, nullptr);
         init();
 
         m_history_entry = History::UserRequest::prefilled();
@@ -538,9 +535,6 @@ namespace mamba
             Console::instance().json_down("actions");
             Console::instance().json_write({ { "PREFIX", Context::instance().target_prefix } });
         }
-        queue_free(&q);
-        queue_free(&decision);
-        queue_free(&job);
 
         m_transaction_context = TransactionContext(
             Context::instance().target_prefix, find_python_version(), specs_to_install);

libmambapy/src/main.cpp

@@ -66,7 +66,11 @@ PYBIND11_MODULE(bindings, m)
     py::class_<MPool>(m, "Pool")
         .def(py::init<>())
         .def("set_debuglevel", &MPool::set_debuglevel)
-        .def("create_whatprovides", &MPool::create_whatprovides);
+        .def("create_whatprovides", &MPool::create_whatprovides)
+        .def("select_solvables", &MPool::select_solvables)
+        .def("matchspec2id", &MPool::matchspec2id)
+        .def("id2pkginfo",
+             [](MPool& self, Id id) { return PackageInfo(pool_id2solvable(self, id)); });
 
     py::class_<MultiPackageCache>(m, "MultiPackageCache")
         .def(py::init<std::vector<fs::path>>())