folkslinux
/
mamba
mirror of https://github.com/mamba-org/mamba.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add optional python_site_packages_path 

Add support for the optional python_site_packages_path field that can
appear in repodata for python packages to specify the location of the
site-packages directory.

This optional field is defined in CEP-17.
This commit is contained in:
Jonathan Helmus 2024-11-06 14:13:45 -06:00 committed by AntoinePrv
parent d5c2a83932
commit e0e925a72f
20 changed files with 201 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
libmamba/ext/solv-cpp/include/solv-cpp/solvable.hpp
View File

@ -53,6 +53,7 @@ namespace solv
auto build_string() const -> std::string_view;
auto file_name() const -> std::string_view;
auto license() const -> std::string_view;
auto python_site_packages_path() const -> std::string_view;
auto md5() const -> std::string_view;
auto noarch() const -> std::string_view;
auto sha256() const -> std::string_view;
@ -168,6 +169,17 @@ namespace solv
void set_license(raw_str_view str) const;
void set_license(const std::string& str) const;
/**
* Set the python_site_packages_path of the solvable.
*
* This is not used by libsolv and is purely for data storing.
*
* @note A call to @ref ObjRepoView::internalize is required for this attribute to
* be available for lookup.
*/
void set_python_site_packages_path(raw_str_view str) const;
void set_python_site_packages_path(const std::string& str) const;
/**
* Set the md5 hash of the solvable file.
*

16
libmamba/ext/solv-cpp/src/solvable.cpp
View File

@ -206,6 +206,22 @@ namespace solv
return set_license(str.c_str());
}
auto ObjSolvableViewConst::python_site_packages_path() const -> std::string_view
{
return ptr_to_strview(::solvable_lookup_str(const_cast<::Solvable*>(raw()), SOLVABLE_MEDIABASE)
);
}
void ObjSolvableView::set_python_site_packages_path(raw_str_view str) const
{
::solvable_set_str(raw(), SOLVABLE_MEDIABASE, str);
}
void ObjSolvableView::set_python_site_packages_path(const std::string& str) const
{
return set_python_site_packages_path(str.c_str());
}
auto ObjSolvableViewConst::md5() const -> std::string_view
{
::Id type = 0;

3
libmamba/ext/solv-cpp/tests/src/test_solvable.cpp
View File

@ -43,6 +43,7 @@ namespace
solv.set_build_string("build");
solv.set_file_name("file.tar.gz");
solv.set_license("MIT");
solv.set_python_site_packages_path("dummy_pspp");
solv.set_md5("6f29ba77e8b03b191c9d667f331bf2a0");
solv.set_sha256("ecde63af23e0d49c0ece19ec539d873ea408a6f966d3126994c6d33ae1b9d3f7");
solv.set_signatures(
@ -62,6 +63,7 @@ namespace
REQUIRE(solv.build_string() == "");
REQUIRE(solv.file_name() == "");
REQUIRE(solv.license() == "");
REQUIRE(solv.python_site_packages_path() == "");
REQUIRE(solv.md5() == "");
REQUIRE(solv.sha256() == "");
REQUIRE(solv.signatures() == "");
@ -83,6 +85,7 @@ namespace
REQUIRE(solv.build_string() == "build");
REQUIRE(solv.file_name() == "file.tar.gz");
REQUIRE(solv.license() == "MIT");
REQUIRE(solv.python_site_packages_path() == "dummy_pspp");
REQUIRE(solv.md5() == "6f29ba77e8b03b191c9d667f331bf2a0");
REQUIRE(
solv.sha256() == "ecde63af23e0d49c0ece19ec539d873ea408a6f966d3126994c6d33ae1b9d3f7"

1
libmamba/include/mamba/core/transaction.hpp
View File

@ -83,6 +83,7 @@ namespace mamba
solver::Solution m_solution;
std::pair<std::string, std::string> m_py_versions;
std::string m_python_site_packages_path;
std::vector<specs::MatchSpec> m_requested_specs;
MTransaction(const CommandParams& command_params, MultiPackageCache&);

1
libmamba/include/mamba/specs/package_info.hpp
View File

@ -47,6 +47,7 @@ namespace mamba::specs
std::string license = {};
std::string md5 = {};
std::string sha256 = {};
std::string python_site_packages_path = {};
std::string signatures = {};
std::vector<std::string> track_features = {};
std::vector<std::string> dependencies = {};

3
libmamba/include/mamba/specs/repo_data.hpp
View File

@ -63,6 +63,9 @@ namespace mamba::specs
/** Optionally a SHA256 hash of the package archive. */
std::optional<std::string> sha256 = {};
/** Optionally a path to the site-packages directory. */
std::optional<std::string> python_site_packages_path = {};
/** A deprecated md5 hash. */
std::optional<std::string> legacy_bz2_md5 = {};

5
libmamba/src/core/query.cpp
View File

@ -325,6 +325,11 @@ namespace mamba
fmt::print(out, fmtstring, "Track Features", fmt::join(pkg.track_features, ","));
}
if (!pkg.python_site_packages_path.empty())
{
fmt::print(out, fmtstring, "Site-packages", pkg.python_site_packages_path);
}
// std::cout << fmt::format<char>(
// " {:<15} {:%Y-%m-%d %H:%M:%S} UTC\n", "Timestamp", fmt::gmtime(pkg.timestamp));

36
libmamba/src/core/transaction.cpp
View File

@ -135,6 +135,32 @@ namespace mamba
return { std::move(new_py_ver), std::move(installed_py_ver) };
}
auto find_python_site_packages_path(
const solver::Solution& solution,
const solver::libsolv::Database& database
) -> std::string
{
// We need to find the python version that will be there after this
// Transaction is finished in order to compile the noarch packages correctly,
// We need to look into installed packages in case we are not installing a new python
// version but keeping the current one.
// Could also be written in term of PrefixData.
std::string python_site_packages_path = {};
if (auto pkg = installed_python(database))
{
python_site_packages_path = pkg->python_site_packages_path;
LOG_INFO << "Found python in installed packages " << python_site_packages_path;
}
if (auto py = solver::find_new_python_in_solution(solution))
{
python_site_packages_path = py->get().python_site_packages_path;
}
return { python_site_packages_path };
}
}
MTransaction::MTransaction(const CommandParams& command_params, MultiPackageCache& caches)
@ -214,6 +240,7 @@ namespace mamba
}
m_py_versions = find_python_version(m_solution, database);
m_python_site_packages_path = find_python_site_packages_path(m_solution, database);
}
MTransaction::MTransaction(
@ -259,6 +286,7 @@ namespace mamba
);
m_py_versions = find_python_version(m_solution, database);
m_python_site_packages_path = find_python_site_packages_path(m_solution, database);
// if no action required, don't even start logging them
if (!empty())
@ -304,6 +332,7 @@ namespace mamba
);
m_py_versions = find_python_version(m_solution, database);
m_python_site_packages_path = find_python_site_packages_path(m_solution, database);
}
class TransactionRollback
@ -395,7 +424,12 @@ namespace mamba
};
TransactionRollback rollback;
TransactionContext transaction_context(ctx.transaction_params(), m_py_versions, m_requested_specs);
TransactionContext transaction_context(
ctx.transaction_params(),
m_py_versions,
m_python_site_packages_path,
m_requested_specs
);
for (const specs::PackageInfo& pkg : m_solution.packages_to_remove())
{

37
libmamba/src/core/transaction_context.cpp
View File

@ -9,12 +9,11 @@
#include <reproc++/drain.hpp>
#include "mamba/core/error_handling.hpp"
#include "mamba/core/output.hpp"
#include "mamba/util/environment.hpp"
#include "mamba/util/string.hpp"
#include "./transaction_context.hpp"
#include "transaction_context.hpp"
extern const char data_compile_pyc_py[];
@ -91,17 +90,28 @@ namespace mamba
}
}
TransactionContext::PythonParams
build_python_params(std::pair<std::string, std::string> py_versions)
TransactionContext::PythonParams build_python_params(
std::pair<std::string, std::string> py_versions,
std::string python_site_packages_path
)
{
TransactionContext::PythonParams res;
if (py_versions.first.size() != 0)
if (py_versions.first.empty())
{
return {};
}
TransactionContext::PythonParams res;
res.has_python = true;
res.python_version = std::move(py_versions.first);
res.old_python_version = std::move(py_versions.second);
res.short_python_version = compute_short_python_version(res.python_version);
res.python_path = get_python_short_path(res.short_python_version);
if (!python_site_packages_path.empty())
{
res.site_packages_path = python_site_packages_path;
}
else
{
res.has_python = true;
res.python_version = std::move(py_versions.first);
res.old_python_version = std::move(py_versions.second);
res.short_python_version = compute_short_python_version(res.python_version);
res.python_path = get_python_short_path(res.short_python_version);
res.site_packages_path = get_python_site_packages_short_path(res.short_python_version);
}
return res;
@ -110,10 +120,13 @@ namespace mamba
TransactionContext::TransactionContext(
TransactionParams transaction_params,
std::pair<std::string, std::string> py_versions,
std::string python_site_packages_path,
std::vector<specs::MatchSpec> lrequested_specs
)
: m_transaction_params(std::move(transaction_params))
, m_python_params(build_python_params(std::move(py_versions)))
, m_python_params(
build_python_params(std::move(py_versions), std::move(python_site_packages_path))
)
, m_requested_specs(std::move(lrequested_specs))
{
if (m_python_params.python_version.size() == 0)

1
libmamba/src/core/transaction_context.hpp
View File

@ -45,6 +45,7 @@ namespace mamba
TransactionContext(
TransactionParams transaction_params,
std::pair<std::string, std::string> py_versions,
std::string python_site_packages_path,
std::vector<specs::MatchSpec> requested_specs
);

11
libmamba/src/solver/libsolv/helpers.cpp
View File

@ -77,6 +77,7 @@ namespace mamba::solver::libsolv
);
solv.set_md5(pkg.md5);
solv.set_sha256(pkg.sha256);
solv.set_python_site_packages_path(pkg.python_site_packages_path);
for (const auto& dep : pkg.dependencies)
{
@ -122,6 +123,7 @@ namespace mamba::solver::libsolv
out.timestamp = s.timestamp();
out.md5 = s.md5();
out.sha256 = s.sha256();
out.python_site_packages_path = s.python_site_packages_path();
out.signatures = s.signatures();
const auto dep_to_str = [&pool](solv::DependencyId id)
@ -288,6 +290,15 @@ namespace mamba::solver::libsolv
solv.set_sha256(std::string(sha256.get_string().value_unsafe()));
}
if (auto python_site_packages_path = pkg["python_site_packages_path"];
!python_site_packages_path.error())
{
auto buffer = std::string(python_site_packages_path.get_string().value_unsafe()
);
solv.set_python_site_packages_path(buffer);
}
if (auto elem = pkg["noarch"]; !elem.error())
{
if (auto noarch = elem.get_bool(); !noarch.error() && noarch.value_unsafe())

10
libmamba/src/specs/package_info.cpp
View File

@ -417,6 +417,10 @@ namespace mamba::specs
{
return invoke_field_string(*this, &PackageInfo::license);
}
if (field_name == "python_site_packages_path")
{
return invoke_field_string(*this, &PackageInfo::python_site_packages_path);
}
if (field_name == "size")
{
return invoke_field_string(*this, &PackageInfo::size);
@ -451,6 +455,7 @@ namespace mamba::specs
p.dependencies,
p.constrains,
p.signatures,
p.python_site_packages_path,
p.defaulted_keys
);
}
@ -497,6 +502,10 @@ namespace mamba::specs
{
j["signatures"] = pkg.signatures;
}
if (!pkg.python_site_packages_path.empty())
{
j["python_site_packages_path"] = pkg.python_site_packages_path;
}
if (pkg.dependencies.empty())
{
j["depends"] = nlohmann::json::array();
@ -539,6 +548,7 @@ namespace mamba::specs
pkg.md5 = j.value("md5", "");
pkg.sha256 = j.value("sha256", "");
pkg.signatures = j.value("signatures", "");
pkg.python_site_packages_path = j.value("python_site_packages_path", "");
if (auto it = j.find("track_features"); it != j.end())
{
if (it->is_string() && !it->get<std::string_view>().empty())

2
libmamba/src/specs/repo_data.cpp
View File

@ -22,6 +22,7 @@ namespace mamba::specs
j["subdir"] = p.subdir;
j["md5"] = p.md5;
j["sha256"] = p.sha256;
j["python_site_packages_path"] = p.python_site_packages_path;
j["legacy_bz2_md5"] = p.legacy_bz2_md5;
j["legacy_bz2_size"] = p.legacy_bz2_size;
j["size"] = p.size;
@ -50,6 +51,7 @@ namespace mamba::specs
deserialize_maybe_missing(j, "subdir", p.subdir);
deserialize_maybe_missing(j, "md5", p.md5);
deserialize_maybe_missing(j, "sha256", p.sha256);
deserialize_maybe_missing(j, "python_site_packages_path", p.python_site_packages_path);
deserialize_maybe_missing(j, "legacy_bz2_md5", p.legacy_bz2_md5);
deserialize_maybe_missing(j, "legacy_bz2_size", p.legacy_bz2_size);
deserialize_maybe_missing(j, "size", p.size);

5
libmamba/tests/src/specs/test_package_info.cpp
View File

@ -143,6 +143,7 @@ namespace
pkg.platform = "linux-64";
pkg.filename = "foo-4.0-mybld.conda";
pkg.license = "MIT";
pkg.python_site_packages_path = "lib/python3.99t/site-packages";
pkg.size = 3200;
pkg.timestamp = 4532;
pkg.sha256 = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b";
@ -167,6 +168,7 @@ namespace
REQUIRE(pkg.field("subdir") == "linux-64");
REQUIRE(pkg.field("filename") == "foo-4.0-mybld.conda");
REQUIRE(pkg.field("license") == "MIT");
REQUIRE(pkg.field("python_site_packages_path") == "lib/python3.99t/site-packages");
REQUIRE(pkg.field("size") == "3200");
REQUIRE(pkg.field("timestamp") == "4532");
}
@ -184,6 +186,7 @@ namespace
REQUIRE(j.at("subdir") == "linux-64");
REQUIRE(j.at("fn") == "foo-4.0-mybld.conda");
REQUIRE(j.at("license") == "MIT");
REQUIRE(j.at("python_site_packages_path") == "lib/python3.99t/site-packages");
REQUIRE(j.at("size") == 3200);
REQUIRE(j.at("timestamp") == 4532);
REQUIRE(
@ -212,6 +215,7 @@ namespace
j["subdir"] = "linux-64";
j["fn"] = "foo-4.0-mybld.conda";
j["license"] = "MIT";
j["python_site_packages_path"] = "lib/python3.99t/site-packages";
j["size"] = 3200;
j["timestamp"] = 4532;
j["sha256"] = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b";
@ -273,6 +277,7 @@ namespace
pkg2.platform = "linux-64";
pkg2.filename = "foo-4.0-mybld.conda";
pkg2.license = "MIT";
pkg2.python_site_packages_path = "lib/python3.99t/site-packages";
pkg2.size = 3200;
pkg2.timestamp = 4532;
pkg2.sha256 = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b";

4
libmamba/tests/src/specs/test_repo_data.cpp
View File

@ -24,6 +24,7 @@ namespace
p.version = Version::parse("1.0.0").value();
p.build_string = "bld";
p.build_number = 3;
p.python_site_packages_path = "dummpy_pspp";
p.subdir = "linux";
p.md5 = "ffsd";
p.noarch = NoArchType::Python;
@ -33,6 +34,7 @@ namespace
REQUIRE(j.at("version") == p.version.to_string());
REQUIRE(j.at("build") == p.build_string);
REQUIRE(j.at("build_number") == p.build_number);
REQUIRE(j.at("python_site_packages_path") == p.python_site_packages_path);
REQUIRE(j.at("subdir") == p.subdir);
REQUIRE(j.at("md5") == p.md5);
REQUIRE(j.at("sha256").is_null());
@ -46,6 +48,7 @@ namespace
j["version"] = "1.1.0";
j["build"] = "foo1";
j["build_number"] = 2;
j["python_site_packages_path"] = "dummy_pspp";
j["subdir"] = "linux";
j["platform"] = nullptr;
j["depends"] = nl::json::array({ "libsolv>=1.0" });
@ -59,6 +62,7 @@ namespace
REQUIRE(j.at("build") == p.build_string);
REQUIRE(j.at("build_number") == p.build_number);
REQUIRE(j.at("subdir") == p.subdir);
REQUIRE(j.at("python_site_packages_path") == p.python_site_packages_path);
REQUIRE_FALSE(p.md5.has_value());
REQUIRE_FALSE(p.platform.has_value());
REQUIRE(p.depends == decltype(p.depends){ "libsolv>=1.0" });

5
libmambapy/src/libmambapy/bindings/specs.cpp
View File

@ -662,6 +662,7 @@ namespace mambapy
decltype(PackageInfo::platform) platform,
decltype(PackageInfo::filename) filename,
decltype(PackageInfo::license) license,
decltype(PackageInfo::python_site_packages_path) python_site_packages_path,
decltype(PackageInfo::md5) md5,
decltype(PackageInfo::sha256) sha256,
decltype(PackageInfo::signatures) signatures,
@ -683,6 +684,7 @@ namespace mambapy
pkg.platform = std::move(platform);
pkg.filename = std::move(filename);
pkg.license = std::move(license);
pkg.python_site_packages_path = std::move(python_site_packages_path);
pkg.md5 = std::move(md5);
pkg.sha256 = std::move(sha256);
pkg.signatures = std::move(signatures);
@ -705,6 +707,8 @@ namespace mambapy
py::arg("platform") = decltype(PackageInfo::platform)(),
py::arg("filename") = decltype(PackageInfo::filename)(),
py::arg("license") = decltype(PackageInfo::license)(),
py::arg("python_site_packages_path") = decltype(PackageInfo::python_site_packages_path)(
),
py::arg("md5") = decltype(PackageInfo::md5)(),
py::arg("sha256") = decltype(PackageInfo::sha256)(),
py::arg("signatures") = decltype(PackageInfo::signatures)(),
@ -740,6 +744,7 @@ namespace mambapy
{ throw std::runtime_error("'fn' has been renamed 'filename'"); }
)
.def_readwrite("license", &PackageInfo::license)
.def_readwrite("python_site_packages_path", &PackageInfo::python_site_packages_path)
.def_readwrite("size", &PackageInfo::size)
.def_readwrite("timestamp", &PackageInfo::timestamp)
.def_readwrite("md5", &PackageInfo::md5)

2
libmambapy/tests/test_specs.py
View File

@ -812,6 +812,8 @@ def test_PackageInfo():
assert pkg.filename == "foo-4.0-mybld.conda"
pkg.license = "MIT"
assert pkg.license == "MIT"
pkg.python_site_packages_path = "lib/python3.99t/site-packages"
assert pkg.python_site_packages_path == "lib/python3.99t/site-packages"
pkg.size = 3200
assert pkg.size == 3200
pkg.timestamp = 4532

25
micromamba/tests/test_create.py
View File

@ -1213,6 +1213,7 @@ def test_set_platform(tmp_home, tmp_root_prefix):
"version,build,cache_tag",
[
["3.10", "*_cpython", "cpython-310"],
["3.13", "*_cp313t", "cpython-313"],
# FIXME: https://github.com/mamba-org/mamba/issues/1432
# [ "3.7", "*_pypy","pypy37"],
],
@ -1227,7 +1228,10 @@ def test_pyc_compilation(tmp_home, tmp_root_prefix, version, build, cache_tag):
if version == "2.7":
cmd += ["-c", "defaults"] # for vc=9.*
else:
site_packages = env_prefix / "lib" / f"python{version}" / "site-packages"
if build.endswith("t"):
site_packages = env_prefix / "lib" / f"python{version}t" / "site-packages"
else:
site_packages = env_prefix / "lib" / f"python{version}" / "site-packages"
if cache_tag:
pyc_fn = Path("__pycache__") / f"six.{cache_tag}.pyc"
@ -1261,6 +1265,25 @@ def test_create_check_dirs(tmp_home, tmp_root_prefix):
assert os.path.isdir(env_prefix / "lib" / "python3.8" / "site-packages" / "traitlets")
@pytest.mark.parametrize("shared_pkgs_dirs", [True], indirect=True)
def test_create_python_site_packages_path(tmp_home, tmp_root_prefix):
env_name = "myenv"
env_prefix = tmp_root_prefix / "envs" / env_name
# imagesize is a noarch: python package
cmd = ["-n", env_name, "python=3.13", "python-freethreading", "imagesize=1.4.1"]
helpers.create(*cmd)
assert os.path.isdir(env_prefix)
if platform.system() == "Windows":
assert os.path.isdir(env_prefix / "lib" / "site-packages" / "imagesize")
else:
# check that the noarch: python package installs into the python_site_packages_path directory
assert os.path.isdir(env_prefix / "lib" / "python3.13t" / "site-packages" / "imagesize")
# and not into the "standard" site-packages directory
assert not os.path.isdir(env_prefix / "lib" / "python3.13" / "site-packages" / "imagesize")
@pytest.mark.parametrize("shared_pkgs_dirs", [True], indirect=True)
@pytest.mark.parametrize("env_file", env_files)
def test_requires_pip_install(tmp_home, tmp_root_prefix, env_file):

16
micromamba/tests/test_install.py
View File

@ -665,6 +665,22 @@ def test_install_check_dirs(tmp_home, tmp_root_prefix):
assert os.path.isdir(env_prefix / "lib" / "python3.8" / "site-packages")
def test_install_python_site_packages_path(tmp_home, tmp_root_prefix):
env_name = "myenv"
env_prefix = tmp_root_prefix / "envs" / env_name
helpers.create("-n", env_name, "python=3.13", "python-freethreading")
helpers.install("-n", env_name, "imagesize")
if helpers.platform.system() == "Windows":
assert os.path.isdir(env_prefix / "lib" / "site-packages" / "imagesize")
else:
# check that the noarch: python package installs into the python_site_packages_path directory
assert os.path.isdir(env_prefix / "lib" / "python3.13t" / "site-packages" / "imagesize")
# and not into the "standard" site-packages directory
assert not os.path.isdir(env_prefix / "lib" / "python3.13" / "site-packages" / "imagesize")
@pytest.mark.parametrize("output_flag", ["", "--json", "--quiet"])
def test_install_check_logs(tmp_home, tmp_root_prefix, output_flag):
env_name = "env-install-check-logs"

20
micromamba/tests/test_repoquery.py
View File

@ -238,3 +238,23 @@ def test_remote_search_not_installed_pkg(yaml_env: Path):
assert "conda-forge" in res["result"]["pkgs"][0]["channel"]
assert res["result"]["pkgs"][0]["name"] == "xtensor"
assert res["result"]["pkgs"][0]["version"] == "0.24.5"
@pytest.mark.parametrize("shared_pkgs_dirs", [True], indirect=True)
def test_remote_search_python_site_packages_path(yaml_env: Path):
res = helpers.umamba_repoquery(
"search",
"-c",
"conda-forge",
"--platform",
"linux-64",
"python=3.13.1=h9a34b6e_5_cp313t",
"--json",
)
assert res["query"]["query"] == "python=3.13.1=h9a34b6e_5_cp313t"
assert res["query"]["type"] == "search"
assert "conda-forge" in res["result"]["pkgs"][0]["channel"]
assert res["result"]["pkgs"][0]["name"] == "python"
assert res["result"]["pkgs"][0]["version"] == "3.13.1"
assert res["result"]["pkgs"][0]["python_site_packages_path"] == "lib/python3.13t/site-packages"