folkslinux
/
mamba
mirror of https://github.com/mamba-org/mamba.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add pip to create env, silence code signing (#885)

This commit is contained in:
Wolf Vollprecht 2021-04-22 18:26:43 +02:00 committed by GitHub
parent 117b8907df
commit e12a0f2474
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 327 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,5 +1,5 @@
include/mamba/core/version.hpp
.DS_Store
.cache
*build

11
include/mamba/api/install.hpp
View File

@ -40,6 +40,17 @@ namespace mamba
MRepo create_repo_from_pkgs_dir(MPool& pool, const fs::path& pkgs_dir);
bool download_explicit(const std::vector<PackageInfo>& pkgs);
struct yaml_file_contents
{
std::string name;
std::vector<std::string> dependencies, channels;
std::vector<std::tuple<std::string, std::vector<std::string>>> other_pkg_mgr_specs;
};
bool eval_selector(const std::string& selector);
yaml_file_contents read_yaml_file(fs::path yaml_file);
}
}

2
include/mamba/core/link.hpp
View File

@ -24,6 +24,8 @@
namespace mamba
{
std::string replace_long_shebang(const std::string& shebang);
std::tuple<std::vector<std::string>, std::unique_ptr<TemporaryFile>> prepare_wrapped_call(
const fs::path& prefix, const std::vector<std::string>& cmd);
struct python_entry_point_parsed
{

238
src/api/install.cpp
View File

@ -23,6 +23,179 @@
namespace mamba
{
static std::vector<std::tuple<std::string, std::vector<std::string>>> other_pkg_mgr_specs;
static std::map<std::string, std::string> other_pkg_mgr_install_instructions
= { { "pip", "pip install {0} --no-input" } };
auto install_for_other_pkgmgr(const std::string& pkg_mgr, const std::vector<std::string>& deps)
{
std::string install_instructions = other_pkg_mgr_install_instructions[pkg_mgr];
replace_all(install_instructions, "{0}", join(" ", deps));
const auto& ctx = Context::instance();
std::vector<std::string> install_args = split(install_instructions, " ");
install_args[0]
= (ctx.target_prefix / get_bin_directory_short_path() / install_args[0]).string();
auto [wrapped_command, tmpfile] = prepare_wrapped_call(ctx.target_prefix, install_args);
reproc::options options;
options.redirect.parent = true;
std::string cwd = ctx.target_prefix;
options.working_directory = cwd.c_str();
std::cout << "\n"
<< termcolor::cyan << "Installing " << pkg_mgr << " packages: " << join(" ", deps)
<< termcolor::reset << std::endl;
LOG_INFO << "Calling: " << join(" ", install_args);
auto [_, ec] = reproc::run(wrapped_command, options);
if (ec)
{
throw std::runtime_error(ec.message());
}
}
auto& truthy_values()
{
static std::map<std::string, int> vals{
{ "win", 0 },
{ "unix", 0 },
{ "osx", 0 },
{ "linux", 0 },
};
const auto& ctx = Context::instance();
if (starts_with(ctx.platform, "win"))
{
vals["win"] = true;
}
else
{
vals["unix"] = true;
if (starts_with(ctx.platform, "linux"))
{
vals["linux"] = true;
}
else if (starts_with(ctx.platform, "osx"))
{
vals["osx"] = true;
}
}
return vals;
}
namespace detail
{
bool eval_selector(const std::string& selector)
{
if (!(starts_with(selector, "sel(") && selector[selector.size() - 1] == ')'))
{
throw std::runtime_error(
"Couldn't parse selector. Needs to start with sel( and end with )");
}
std::string expr = selector.substr(4, selector.size() - 5);
if (truthy_values().find(expr) == truthy_values().end())
{
throw std::runtime_error("Couldn't parse selector. Value not in [unix, linux, "
"osx, win] or additional whitespaces found.");
}
return truthy_values()[expr];
}
yaml_file_contents read_yaml_file(fs::path yaml_file)
{
yaml_file_contents result;
YAML::Node f;
try
{
f = YAML::LoadFile(yaml_file);
}
catch (YAML::Exception& e)
{
LOG_ERROR << "Error in spec file: " << yaml_file;
}
YAML::Node deps = f["dependencies"];
YAML::Node final_deps;
std::vector<std::string> pip_deps;
for (auto it = deps.begin(); it != deps.end(); ++it)
{
if (it->IsScalar())
{
final_deps.push_back(*it);
}
else if (it->IsMap())
{
// we merge a map to the upper level if the selector works
for (const auto& map_el : *it)
{
std::string key = map_el.first.as<std::string>();
if (starts_with(key, "sel("))
{
bool selected = detail::eval_selector(key);
if (selected)
{
const YAML::Node& rest = map_el.second;
if (rest.IsScalar())
{
final_deps.push_back(rest);
}
else
{
throw std::runtime_error(
"Complicated selection merge not implemented yet.");
}
}
}
else if (key == "pip")
{
result.other_pkg_mgr_specs.push_back(std::make_tuple(
std::string("pip"), map_el.second.as<std::vector<std::string>>()));
pip_deps = map_el.second.as<std::vector<std::string>>();
}
}
}
}
std::vector<std::string> dependencies = final_deps.as<std::vector<std::string>>();
result.dependencies = dependencies;
if (f["channels"])
{
try
{
result.channels = f["channels"].as<std::vector<std::string>>();
}
catch (YAML::Exception& e)
{
throw std::runtime_error(mamba::concat(
"Could not read 'channels' as list of strings from ", yaml_file.string()));
}
}
else
{
LOG_DEBUG << "No 'channels' specified in file: " << yaml_file;
}
if (f["name"])
{
result.name = f["name"].as<std::string>();
}
{
LOG_DEBUG << "No env 'name' specified in file: " << yaml_file;
}
return result;
}
}
void install()
{
auto& config = Configuration::instance();
@ -263,6 +436,11 @@ namespace mamba
detail::create_target_directory(ctx.target_prefix);
}
trans.execute(prefix_data);
for (const auto& [pkg_mgr, deps] : other_pkg_mgr_specs)
{
mamba::install_for_other_pkgmgr(pkg_mgr, deps);
}
}
}
@ -364,82 +542,44 @@ namespace mamba
// read specs from file :)
if (ends_with(file, ".yml") || ends_with(file, ".yaml"))
{
YAML::Node f;
try
{
f = YAML::LoadFile(file);
}
catch (YAML::Exception& e)
{
LOG_ERROR << "Error in spec file: " << file;
continue;
}
auto parse_result = read_yaml_file(file);
if (f["channels"])
if (parse_result.channels.size() != 0)
{
std::vector<std::string> yaml_channels;
try
{
yaml_channels = f["channels"].as<std::vector<std::string>>();
}
catch (YAML::Exception& e)
{
throw std::runtime_error(mamba::concat(
"Could not read 'channels' as list of strings from ", file));
}
YAML::Node updated_channels;
if (channels.cli_configured())
{
updated_channels = channels.cli_yaml_value();
}
for (auto& c : yaml_channels)
for (auto& c : parse_result.channels)
{
updated_channels.push_back(c);
}
channels.set_cli_yaml_value(updated_channels);
}
else
if (parse_result.name.size() != 0)
{
LOG_DEBUG << "No 'channels' specified in file: " << file;
env_name.set_cli_yaml_value(parse_result.name);
}
if (f["name"])
if (parse_result.dependencies.size() != 0)
{
env_name.set_cli_yaml_value(f["name"]);
}
{
LOG_DEBUG << "No env 'name' specified in file: " << file;
}
if (f["dependencies"])
{
std::vector<std::string> yaml_specs;
try
{
yaml_specs = f["dependencies"].as<std::vector<std::string>>();
}
catch (YAML::Exception& e)
{
throw std::runtime_error(mamba::concat(
"Could not read 'dependencies' as list of strings from ", file));
}
YAML::Node updated_specs;
if (specs.cli_configured())
{
updated_specs = specs.cli_yaml_value();
}
for (auto& s : yaml_specs)
for (auto& s : parse_result.dependencies)
{
updated_specs.push_back(s);
}
specs.set_cli_yaml_value(updated_specs);
}
else
if (parse_result.other_pkg_mgr_specs.size())
{
throw std::runtime_error(
concat("No 'dependencies' specified in file: ", file));
other_pkg_mgr_specs = parse_result.other_pkg_mgr_specs;
}
}
else

14
src/core/link.cpp
View File

@ -474,7 +474,8 @@ namespace mamba
return tf;
}
auto prepare_wrapped_call(const fs::path& prefix, const std::vector<std::string>& cmd)
std::tuple<std::vector<std::string>, std::unique_ptr<TemporaryFile>> prepare_wrapped_call(
const fs::path& prefix, const std::vector<std::string>& cmd)
{
std::vector<std::string> command_args;
std::unique_ptr<TemporaryFile> script_file;
@ -884,9 +885,18 @@ namespace mamba
#if defined(__APPLE__)
if (binary_changed && m_pkg_info.subdir == "osx-arm64")
{
reproc::options options;
if (Context::instance().verbosity <= 1)
{
reproc::redirect silence;
silence.type = reproc::redirect::discard;
options.redirect.out = silence;
options.redirect.err = silence;
}
std::vector<std::string> cmd
= { "/usr/bin/codesign", "-s", "-", "-f", dst.string() };
auto [status, ec] = reproc::run(cmd, reproc::options{});
auto [status, ec] = reproc::run(cmd, options);
if (ec)
{
throw std::runtime_error(std::string("Could not codesign executable")

18
test/CMakeLists.txt
View File

@ -26,16 +26,22 @@ set(TEST_SRCS
test_graph.cpp
test_pinning.cpp
test_virtual_packages.cpp
test_env_file_reading.cpp
)
add_executable(test_mamba ${TEST_SRCS})
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/history_test/conda-meta/history
${CMAKE_CURRENT_BINARY_DIR}/history_test/conda-meta/history COPYONLY)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/history_test/conda-meta/aux_file
${CMAKE_CURRENT_BINARY_DIR}/history_test/conda-meta/aux_file COPYONLY)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config_test/.condarc
${CMAKE_CURRENT_BINARY_DIR}/config_test/.condarc COPYONLY)
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/history_test/conda-meta/history
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/history_test/conda-meta/)
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/history_test/conda-meta/aux_file
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/history_test/conda-meta/)
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/config_test/.condarc
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/config_test/)
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/config_test/.condarc
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/config_test/)
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/env_file_test
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
target_link_libraries(test_mamba PRIVATE GTest::GTest GTest::Main ${CMAKE_THREAD_LIBS_INIT})
target_link_libraries(test_mamba PUBLIC mamba-static)

6
test/env_file_test/env_1.yaml Normal file
View File

@ -0,0 +1,6 @@
name: env_1
channels: [conda-forge, bioconda]
dependencies:
- test1
- test2
- test3

11
test/env_file_test/env_2.yaml Normal file
View File

@ -0,0 +1,11 @@
name: env_2
channels:
- conda-forge
- bioconda
dependencies:
- sel(win): test1-win
- sel(unix): test1-unix
- sel(linux): test1-linux
- sel(linux): test2-linux
- sel(osx): test1-osx
- test4

9
test/env_file_test/env_3.yaml Normal file
View File

@ -0,0 +1,9 @@
name: env_3
channels: [conda-forge, bioconda]
dependencies:
- test1
- test2
- test3
- pip:
- pytest
- numpy

4
test/test_channel.cpp
View File

@ -11,8 +11,10 @@ namespace mamba
#ifdef __linux__
std::string platform("linux-64");
#elif __APPLE__
#elif __APPLE__ && __x86_64__
std::string platform("osx-64");
#elif __APPLE__ && __arm64__
std::string platform("osx-arm64");
#elif _WIN32
std::string platform("win-64");
#endif

71
test/test_env_file_reading.cpp Normal file
View File

@ -0,0 +1,71 @@
#include <gtest/gtest.h>
#include "mamba/api/install.hpp"
namespace mamba
{
TEST(env_file_reading, selector)
{
using namespace detail;
if (on_linux || on_mac)
{
EXPECT_TRUE(eval_selector("sel(unix)"));
if (on_mac)
{
EXPECT_TRUE(eval_selector("sel(osx)"));
EXPECT_FALSE(eval_selector("sel(linux)"));
EXPECT_FALSE(eval_selector("sel(win)"));
}
else
{
EXPECT_TRUE(eval_selector("sel(linux)"));
EXPECT_FALSE(eval_selector("sel(osx)"));
EXPECT_FALSE(eval_selector("sel(win)"));
}
}
else if (on_win)
{
EXPECT_TRUE(eval_selector("sel(win)"));
EXPECT_FALSE(eval_selector("sel(osx)"));
EXPECT_FALSE(eval_selector("sel(linux)"));
}
}
TEST(env_file_reading, specs_selection)
{
using V = std::vector<std::string>;
auto res = detail::read_yaml_file("env_file_test/env_1.yaml");
EXPECT_EQ(res.name, "env_1");
EXPECT_EQ(res.channels, V({ "conda-forge", "bioconda" }));
EXPECT_EQ(res.dependencies, V({ "test1", "test2", "test3" }));
EXPECT_FALSE(res.other_pkg_mgr_specs.size());
auto res2 = detail::read_yaml_file("env_file_test/env_2.yaml");
EXPECT_EQ(res2.name, "env_2");
EXPECT_EQ(res2.channels, V({ "conda-forge", "bioconda" }));
#ifdef __linux__
EXPECT_EQ(res2.dependencies, V({ "test1-unix", "test1-linux", "test2-linux", "test4" }));
#elif __APPLE__
EXPECT_EQ(res2.dependencies, V({ "test1-unix", "test1-osx", "test4" }));
#elif _WIN32
EXPECT_EQ(res2.dependencies, V({ "test1-win", "test4" }));
#endif
EXPECT_FALSE(res2.other_pkg_mgr_specs.size());
}
TEST(env_file_reading, external_pkg_mgrs)
{
using V = std::vector<std::string>;
auto res = detail::read_yaml_file("env_file_test/env_3.yaml");
EXPECT_EQ(res.name, "env_3");
EXPECT_EQ(res.channels, V({ "conda-forge", "bioconda" }));
EXPECT_EQ(res.dependencies, V({ "test1", "test2", "test3" }));
EXPECT_TRUE(res.other_pkg_mgr_specs.size());
EXPECT_EQ(res.other_pkg_mgr_specs.size(), 1);
auto o = res.other_pkg_mgr_specs[0];
EXPECT_EQ(std::get<0>(o), "pip");
EXPECT_EQ(std::get<1>(o), V({ "pytest", "numpy" }));
}
} // namespace mamba