folkslinux
/
mamba
mirror of https://github.com/mamba-org/mamba.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

pip packages support with `list` (#3565)

This commit is contained in:
Hind-M 2024-10-30 10:20:25 +01:00 committed by GitHub
parent 5bfdd90adf
commit 0075a24e4c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 208 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libmamba/include/mamba/core/activation.hpp
View File

@ -258,8 +258,6 @@ namespace mamba
fs::u8path hook_source_path() override;
};
std::vector<fs::u8path> get_path_dirs(const fs::u8path& prefix);
} // namespace mamba
#endif

8
libmamba/include/mamba/core/prefix_data.hpp
View File

@ -28,9 +28,12 @@ namespace mamba
create(const fs::u8path& prefix_path, ChannelContext& channel_context);
void add_packages(const std::vector<specs::PackageInfo>& packages);
const package_map& records() const;
void load_single_record(const fs::u8path& path);
const package_map& records() const;
const package_map& pip_records() const;
package_map all_pkg_mgr_records() const;
History& history();
const fs::u8path& path() const;
std::vector<specs::PackageInfo> sorted_records() const;
@ -44,8 +47,11 @@ namespace mamba
PrefixData(const fs::u8path& prefix_path, ChannelContext& channel_context);
void load_site_packages();
History m_history;
package_map m_package_records;
package_map m_pip_package_records;
fs::u8path m_prefix_path;
ChannelContext& m_channel_context;

1
libmamba/include/mamba/specs/package_info.hpp
View File

@ -68,6 +68,7 @@ namespace mamba::specs
PackageInfo() = default;
explicit PackageInfo(std::string name);
PackageInfo(std::string name, std::string version, std::string build_string, std::size_t build_number);
PackageInfo(std::string name, std::string version, std::string build_string, std::string channel);
[[nodiscard]] auto json_signable() const -> nlohmann::json;
[[nodiscard]] auto str() const -> std::string;

6
libmamba/include/mamba/util/environment.hpp
View File

@ -11,6 +11,7 @@
#include <string>
#include <string_view>
#include <unordered_map>
#include <vector>
#include "mamba/fs/filesystem.hpp"
#include "mamba/util/build.hpp"
@ -93,6 +94,11 @@ namespace mamba::util
*/
[[nodiscard]] constexpr auto pathsep() -> char;
/**
* Return directories of the given prefix path.
*/
[[nodiscard]] auto get_path_dirs(const fs::u8path& prefix) -> std::vector<fs::u8path>;
/**
* Return the full path of a program from its name.
*/

1
libmamba/src/api/install.cpp
View File

@ -10,7 +10,6 @@
#include "mamba/api/channel_loader.hpp"
#include "mamba/api/configuration.hpp"
#include "mamba/api/install.hpp"
#include "mamba/core/activation.hpp"
#include "mamba/core/channel_context.hpp"
#include "mamba/core/context.hpp"
#include "mamba/core/env_lockfile.hpp"

41
libmamba/src/api/list.cpp
View File

@ -67,14 +67,16 @@ namespace mamba
{
regex = '^' + regex + '$';
}
std::regex spec_pat(regex);
auto all_records = std::move(prefix_data.all_pkg_mgr_records());
if (ctx.output_params.json)
{
auto jout = nlohmann::json::array();
std::vector<std::string> keys;
for (const auto& pkg : prefix_data.records())
for (const auto& pkg : all_records)
{
keys.push_back(pkg.first);
}
@ -83,24 +85,33 @@ namespace mamba
for (const auto& key : keys)
{
auto obj = nlohmann::json();
const auto& pkg_info = prefix_data.records().find(key)->second;
const auto& pkg_info = all_records.find(key)->second;
if (regex.empty() || std::regex_search(pkg_info.name, spec_pat))
{
auto channels = channel_context.make_channel(pkg_info.package_url);
assert(channels.size() == 1); // A URL can only resolve to one channel
obj["base_url"] = strip_from_filename_and_platform(
channels.front().url().str(specs::CondaURL::Credentials::Remove),
pkg_info.filename,
pkg_info.platform
);
if (pkg_info.package_url.empty() && (pkg_info.channel == "pypi"))
{
obj["base_url"] = "https://pypi.org/";
obj["channel"] = pkg_info.channel;
}
else
{
obj["base_url"] = strip_from_filename_and_platform(
channels.front().url().str(specs::CondaURL::Credentials::Remove),
pkg_info.filename,
pkg_info.platform
);
obj["channel"] = strip_from_filename_and_platform(
channels.front().display_name(),
pkg_info.filename,
pkg_info.platform
);
}
obj["build_number"] = pkg_info.build_number;
obj["build_string"] = pkg_info.build_string;
obj["channel"] = strip_from_filename_and_platform(
channels.front().display_name(),
pkg_info.filename,
pkg_info.platform
);
obj["dist_name"] = pkg_info.str();
obj["name"] = pkg_info.name;
obj["platform"] = pkg_info.platform;
@ -121,7 +132,7 @@ namespace mamba
auto requested_specs = prefix_data.history().get_requested_specs_map();
// order list of packages from prefix_data by alphabetical order
for (const auto& package : prefix_data.records())
for (const auto& package : all_records)
{
if (regex.empty() || std::regex_search(package.second.name, spec_pat))
{
@ -132,6 +143,10 @@ namespace mamba
{
formatted_pkgs.channel = "";
}
else if (package.second.channel == "pypi")
{
formatted_pkgs.channel = package.second.channel;
}
else
{
auto channels = channel_context.make_channel(package.second.channel);

5
libmamba/src/api/utils.cpp
View File

@ -12,8 +12,7 @@
#include <reproc/reproc.h>
// TODO includes to be removed after moving some functions/structs around
#include "mamba/api/install.hpp" // other_pkg_mgr_spec
#include "mamba/core/activation.hpp" // get_path_dirs
#include "mamba/api/install.hpp" // other_pkg_mgr_spec
#include "mamba/core/context.hpp"
#include "mamba/core/util.hpp"
#include "mamba/fs/filesystem.hpp"
@ -33,7 +32,7 @@ namespace mamba
)
{
const auto get_python_path = [&]
{ return util::which_in("python", get_path_dirs(target_prefix)).string(); };
{ return util::which_in("python", util::get_path_dirs(target_prefix)).string(); };
command_args cmd = { get_python_path(), "-m", "pip", "install" };
command_args cmd_extension = { "-r", spec_file, "--no-input", "--quiet" };

23
libmamba/src/core/activation.cpp
View File

@ -208,23 +208,6 @@ namespace mamba
}
}
std::vector<fs::u8path> get_path_dirs(const fs::u8path& prefix)
{
if (util::on_win)
{
return { prefix,
prefix / "Library" / "mingw-w64" / "bin",
prefix / "Library" / "usr" / "bin",
prefix / "Library" / "bin",
prefix / "Scripts",
prefix / "bin" };
}
else
{
return { prefix / "bin" };
}
}
std::vector<fs::u8path> Activator::get_PATH()
{
std::vector<fs::u8path> path;
@ -271,7 +254,7 @@ namespace mamba
// TODO check if path_conversion does something useful here.
// path_list[0:0] = list(self.path_conversion(self._get_path_dirs(prefix)))
std::vector<fs::u8path> final_path = get_path_dirs(prefix);
std::vector<fs::u8path> final_path = util::get_path_dirs(prefix);
final_path.insert(final_path.end(), path_list.begin(), path_list.end());
final_path.erase(std::unique(final_path.begin(), final_path.end()), final_path.end());
std::string result = util::join(util::pathsep(), final_path).string();
@ -285,7 +268,7 @@ namespace mamba
std::vector<fs::u8path> current_path = get_PATH();
assert(!old_prefix.empty());
std::vector<fs::u8path> old_prefix_dirs = get_path_dirs(old_prefix);
std::vector<fs::u8path> old_prefix_dirs = util::get_path_dirs(old_prefix);
// remove all old paths
std::vector<fs::u8path> cleaned_path;
@ -312,7 +295,7 @@ namespace mamba
std::vector<fs::u8path> final_path;
if (!new_prefix.empty())
{
final_path = get_path_dirs(new_prefix);
final_path = util::get_path_dirs(new_prefix);
final_path.insert(final_path.end(), current_path.begin(), current_path.end());
// remove duplicates

102
libmamba/src/core/prefix_data.cpp
View File

@ -9,11 +9,14 @@
#include <unordered_map>
#include <utility>
#include <reproc++/run.hpp>
#include "mamba/core/channel_context.hpp"
#include "mamba/core/output.hpp"
#include "mamba/core/prefix_data.hpp"
#include "mamba/core/util.hpp"
#include "mamba/specs/conda_url.hpp"
#include "mamba/util/environment.hpp"
#include "mamba/util/graph.hpp"
#include "mamba/util/string.hpp"
@ -56,6 +59,8 @@ namespace mamba
}
}
}
// Load packages installed with pip
load_site_packages();
}
void PrefixData::add_packages(const std::vector<specs::PackageInfo>& packages)
@ -73,6 +78,22 @@ namespace mamba
return m_package_records;
}
const PrefixData::package_map& PrefixData::pip_records() const
{
return m_pip_package_records;
}
PrefixData::package_map PrefixData::all_pkg_mgr_records() const
{
PrefixData::package_map merged_records = m_package_records;
// Note that if the same key (pkg name) is present in both `m_package_records` and
// `m_pip_package_records`, the latter is not considered
// (this may be modified to be completely independent in the future)
merged_records.insert(m_pip_package_records.begin(), m_pip_package_records.end());
return merged_records;
}
std::vector<specs::PackageInfo> PrefixData::sorted_records() const
{
// TODO add_pip_as_python_dependency
@ -166,10 +187,89 @@ namespace mamba
auto channels = m_channel_context.make_channel(prec.channel);
// If someone wrote multichannel names in repodata_record, we don't know which one is the
// correct URL. This is must never happen!
// correct URL. This must never happen!
assert(channels.size() == 1);
using Credentials = specs::CondaURL::Credentials;
prec.channel = channels.front().platform_url(prec.platform).str(Credentials::Remove);
m_package_records.insert({ prec.name, std::move(prec) });
}
// Load python packages installed with pip in the site-packages of the prefix.
void PrefixData::load_site_packages()
{
LOG_INFO << "Loading site packages";
// Look for `pip` package and return if it doesn't exist
auto python_pkg_record = m_package_records.find("pip");
if (python_pkg_record == m_package_records.end())
{
LOG_DEBUG << "`pip` not found";
return;
}
// Run `pip freeze`
std::string out, err;
const auto get_python_path = [&]
{ return util::which_in("python", util::get_path_dirs(m_prefix_path)).string(); };
const auto args = std::array<std::string, 5>{ get_python_path(),
"-m",
"pip",
"inspect",
"--local" };
auto [status, ec] = reproc::run(
args,
reproc::options{},
reproc::sink::string(out),
reproc::sink::string(err)
);
if (ec)
{
throw std::runtime_error(ec.message());
}
// Nothing installed with `pip`
if (out.empty())
{
LOG_DEBUG << "Nothing installed with `pip`";
return;
}
nlohmann::json j = nlohmann::json::parse(out);
if (j.contains("installed") && j["installed"].is_array())
{
for (const auto& package : j["installed"])
{
// Get the package metadata, if requested and installed with `pip`
if (package.contains("requested") && package.contains("installer")
&& package["requested"] == true && package["installer"] == "pip")
{
if (package.contains("metadata"))
{
// NOTE As checking the presence of all used keys in the json object can be
// cumbersome and might affect the code readability, the elements where the
// check with `contains` is skipped are considered mandatory. If a bug is
// ever to occur in the future, checking the relevant key with `contains`
// should be introduced then.
auto prec = specs::PackageInfo(
package["metadata"]["name"],
package["metadata"]["version"],
"pypi_0",
"pypi"
);
// Set platform by concatenating `sys_platform` and `platform_machine` to
// have something equivalent to `conda-forge`
if (j.contains("environment"))
{
prec.platform = j["environment"]["sys_platform"].get<std::string>() + "-"
+ j["environment"]["platform_machine"].get<std::string>();
}
m_pip_package_records.insert({ prec.name, std::move(prec) });
}
}
}
}
}
} // namespace mamba

2
libmamba/src/core/transaction.cpp
View File

@ -362,8 +362,6 @@ namespace mamba
bool
MTransaction::execute(const Context& ctx, ChannelContext& channel_context, PrefixData& prefix)
{
using Solution = solver::Solution;
// JSON output
// back to the top level if any action was required
if (!empty())

8
libmamba/src/specs/package_info.cpp
View File

@ -156,6 +156,14 @@ namespace mamba::specs
{
}
PackageInfo::PackageInfo(std::string n, std::string v, std::string b, std::string c)
: name(std::move(n))
, version(std::move(v))
, build_string(std::move(b))
, channel(std::move(c))
{
}
namespace
{
template <typename T, typename U>

17
libmamba/src/util/environment.cpp
View File

@ -408,6 +408,23 @@ namespace mamba::util
}
}
auto get_path_dirs(const fs::u8path& prefix) -> std::vector<fs::u8path>
{
if (on_win)
{
return { prefix,
prefix / "Library" / "mingw-w64" / "bin",
prefix / "Library" / "usr" / "bin",
prefix / "Library" / "bin",
prefix / "Scripts",
prefix / "bin" };
}
else
{
return { prefix / "bin" };
}
}
auto which(std::string_view exe) -> fs::u8path
{
if (auto paths = get_env("PATH"))

35
micromamba/tests/test_list.py
View File

@ -1,4 +1,6 @@
import platform
import subprocess
import sys
import pytest
@ -40,6 +42,39 @@ def test_list_name(tmp_home, tmp_root_prefix, tmp_xtensor_env, quiet_flag):
assert full_names == ["xtensor"]
env_yaml_content_to_install_numpy_with_pip = """
channels:
- conda-forge
dependencies:
- pip
- pip:
- numpy==1.26.4
"""
@pytest.mark.parametrize("shared_pkgs_dirs", [True], indirect=True)
def test_list_with_pip(tmp_home, tmp_root_prefix, tmp_path):
env_name = "env-list_with_pip"
tmp_root_prefix / "envs" / env_name
env_file_yml = tmp_path / "test_env_yaml_content_to_install_numpy_with_pip.yaml"
env_file_yml.write_text(env_yaml_content_to_install_numpy_with_pip)
helpers.create("-n", env_name, "python=3.12", "--json", no_dry_run=True)
helpers.install("-n", env_name, "-f", env_file_yml, "--json", no_dry_run=True)
res = helpers.umamba_list("-n", env_name, "--json")
assert any(
package["name"] == "numpy"
and package["version"] == "1.26.4"
and package["base_url"] == "https://pypi.org/"
and package["build_string"] == "pypi_0"
and package["channel"] == "pypi"
and package["platform"] == sys.platform + "-" + platform.machine()
for package in res
)
@pytest.mark.parametrize("env_selector", ["name", "prefix"])
@pytest.mark.parametrize("shared_pkgs_dirs", [True], indirect=True)
def test_not_existing(tmp_home, tmp_root_prefix, tmp_xtensor_env, env_selector):