folkslinux
/
mamba
mirror of https://github.com/mamba-org/mamba.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix unicode on win

This commit is contained in:
Wolf Vollprecht 2021-11-26 12:09:05 +01:00
parent 7b51c6f899
commit d97ac17570
28 changed files with 441 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
libmamba/include/mamba/core/fsutil.hpp
View File

@ -71,7 +71,7 @@ namespace mamba
}
}
// directory exists, now create empty file
std::ofstream ostream(path, std::ios::out);
std::ofstream ostream = open_ofstream(path, std::ios::out);
if (ostream.fail())
throw fs::filesystem_error("File creation failed",
std::make_error_code(std::errc::permission_denied));
@ -85,8 +85,7 @@ namespace mamba
if (fs::is_directory(path.parent_path()))
{
bool path_existed = lexists(path);
std::ofstream test;
test.open(path, std::ios_base::out | std::ios_base::app);
std::ofstream test = open_ofstream(path, std::ios_base::out | std::ios_base::app);
bool is_writable = test.is_open();
if (!path_existed)
{

4
libmamba/include/mamba/core/history.hpp
View File

@ -23,7 +23,7 @@ namespace mamba
class History
{
public:
History(const std::string& prefix);
History(const fs::path& prefix);
struct ParseResult
{
@ -66,7 +66,7 @@ namespace mamba
std::unordered_map<std::string, MatchSpec> get_requested_specs_map();
void add_entry(const History::UserRequest& entry);
std::string m_prefix;
fs::path m_prefix;
fs::path m_history_file_path;
};

2
libmamba/include/mamba/core/prefix_data.hpp
View File

@ -21,7 +21,7 @@ namespace mamba
public:
using package_map = std::unordered_map<std::string, PackageInfo>;
PrefixData(const std::string& prefix_path);
PrefixData(const fs::path& prefix_path);
void load();
void add_virtual_packages(const std::vector<PackageInfo>& packages);

6
libmamba/include/mamba/core/repo.hpp
View File

@ -97,7 +97,7 @@ namespace mamba
void add_package_info(Repodata*, const PackageInfo& pkg_info);
void add_pip_as_python_dependency();
const std::string& index_file();
const fs::path& index_file();
std::string name() const;
bool write() const;
@ -109,9 +109,9 @@ namespace mamba
bool clear(bool reuse_ids);
private:
bool read_file(const std::string& filename);
bool read_file(const fs::path& filename);
std::string m_json_file, m_solv_file;
fs::path m_json_file, m_solv_file;
std::string m_url;
RepoMetadata m_metadata;

36
libmamba/include/mamba/core/util.hpp
View File

@ -405,6 +405,42 @@ namespace mamba
std::time_t parse_utc_timestamp(const std::string& timestamp);
void assert_reproc_success(const reproc::options& options, int status, std::error_code ec);
inline std::ofstream open_ofstream(const fs::path& path,
std::ios::openmode mode = std::ios::out | std::ios::binary)
{
std::ofstream outfile;
#if _WIN32
outfile.open(path.wstring(), mode);
#else
outfile.open(path, mode);
#endif
if (!outfile.good())
{
LOG_ERROR << "Error opening " << path << ": " << strerror(errno);
}
return outfile;
}
inline std::ifstream open_ifstream(const fs::path& path,
std::ios::openmode mode = std::ios::in | std::ios::binary)
{
std::ifstream infile;
#if _WIN32
infile.open(path.wstring(), mode);
#else
infile.open(path, mode);
#endif
if (!infile.good())
{
LOG_ERROR << "Error opening " << path << ": " << strerror(errno);
}
return infile;
}
} // namespace mamba
#endif // MAMBA_UTIL_HPP

8
libmamba/include/mamba/core/util_os.hpp
View File

@ -27,6 +27,14 @@ namespace mamba
std::string windows_version();
std::string macos_version();
std::string linux_version();
void init_console();
void reset_console();
#ifdef _WIN32
std::string to_utf8(const wchar_t* w, size_t s);
std::string to_utf8(const wchar_t* w);
#endif
}
#endif

8
libmamba/include/mamba/core/validate.hpp
View File

@ -22,10 +22,10 @@ namespace validate
{
using nlohmann::json;
std::string sha256sum(const std::string& path);
std::string md5sum(const std::string& path);
bool sha256(const std::string& path, const std::string& validation);
bool md5(const std::string& path, const std::string& validation);
std::string sha256sum(const fs::path& path);
std::string md5sum(const fs::path& path);
bool sha256(const fs::path& path, const std::string& validation);
bool md5(const fs::path& path, const std::string& validation);
bool file_size(const fs::path& path, std::uintmax_t validation);
const std::size_t MAMBA_SHA256_SIZE_HEX = 64;

2
libmamba/src/api/install.cpp
View File

@ -41,7 +41,7 @@ namespace mamba
std::string install_instructions = other_pkg_mgr_install_instructions[pkg_mgr];
TemporaryFile specs;
std::ofstream specs_f(specs.path());
std::ofstream specs_f = open_ofstream(specs.path());
for (auto& d : deps)
specs_f << d.c_str() << '\n';
specs_f.close();

2
libmamba/src/core/activation.cpp
View File

@ -898,7 +898,7 @@ namespace mamba
out << "@CALL " << p << "\n";
}
std::ofstream out_file(tempfile_ptr->path());
std::ofstream out_file = open_ofstream(tempfile_ptr->path());
out_file << out.str();
// note: we do not delete the tempfile ptr intentionally, so that the temp
// file stays

4
libmamba/src/core/environments_manager.cpp
View File

@ -53,7 +53,7 @@ namespace mamba
return;
}
std::ofstream out(env_txt_file, std::ios::app);
std::ofstream out = open_ofstream(env_txt_file, std::ios::app);
out << final_location_string << std::endl;
if (out.bad())
{
@ -168,7 +168,7 @@ namespace mamba
}
if (final_lines.size() != lines.size())
{
std::ofstream out(env_txt_file);
std::ofstream out = open_ofstream(env_txt_file);
for (auto& l : final_lines)
{
out << remove_trailing_slash(l) << std::endl;

2
libmamba/src/core/fetch.cpp
View File

@ -281,7 +281,7 @@ namespace mamba
auto* s = reinterpret_cast<DownloadTarget*>(self);
if (!s->m_file.is_open())
{
s->m_file = std::ofstream(s->m_filename, std::ios::binary);
s->m_file = open_ofstream(s->m_filename, std::ios::binary);
if (!s->m_file)
{
LOG_ERROR << "Could not open file for download " << s->m_filename << ": "

29
libmamba/src/core/history.cpp
View File

@ -12,9 +12,9 @@
namespace mamba
{
History::History(const std::string& prefix)
History::History(const fs::path& prefix)
: m_prefix(prefix)
, m_history_file_path(fs::path(m_prefix) / "conda-meta" / "history")
, m_history_file_path(m_prefix / "conda-meta" / "history")
{
}
@ -30,7 +30,8 @@ namespace mamba
}
static std::regex head_re("==>\\s*(.+?)\\s*<==");
std::ifstream in_file(m_history_file_path);
std::ifstream in_file = open_ifstream(m_history_file_path, std::ios::in);
std::string line;
while (getline(in_file, line))
{
@ -46,11 +47,27 @@ namespace mamba
}
else if (line[0] == '#')
{
res[res.size() - 1].comments.push_back(line);
if (res.size() > 0)
{
res[res.size() - 1].comments.push_back(line);
}
else
{
res.push_back(ParseResult());
res[0].comments.push_back(line);
}
}
else if (line.size() != 0)
{
res[res.size() - 1].diff.insert(line);
if (res.size() > 0)
{
res[res.size() - 1].diff.insert(line);
}
else
{
res.push_back(ParseResult());
res[0].diff.insert(line);
}
}
}
return res;
@ -213,7 +230,7 @@ namespace mamba
{
path::touch(m_history_file_path);
}
std::ofstream out(m_history_file_path, std::ios::app);
std::ofstream out = open_ofstream(m_history_file_path, std::ios::app);
if (out.fail())
{

27
libmamba/src/core/link.cpp
View File

@ -145,7 +145,7 @@ namespace mamba
<< fs::relative(script_path, m_context->target_prefix).string() << std::endl;
fs::remove(script_path);
}
std::ofstream out_file(script_path);
std::ofstream out_file = open_ofstream(script_path);
fs::path python_path;
if (m_context->has_python)
@ -180,7 +180,8 @@ namespace mamba
fs::remove(m_context->target_prefix / script_exe);
}
std::ofstream conda_exe_f(m_context->target_prefix / script_exe, std::ios::binary);
std::ofstream conda_exe_f
= open_ofstream(m_context->target_prefix / script_exe, std::ios::binary);
conda_exe_f.write(reinterpret_cast<char*>(conda_exe), conda_exe_len);
conda_exe_f.close();
make_executable(m_context->target_prefix / script_exe);
@ -245,7 +246,7 @@ namespace mamba
fs::create_directories(target_full_path.parent_path());
}
std::ofstream out_file(target_full_path);
std::ofstream out_file = open_ofstream(target_full_path);
out_file << "!#" << python_full_path.c_str() << "\n";
application_entry_point_template(out_file, win_path_double_escape(source_full_path));
out_file.close();
@ -283,7 +284,7 @@ namespace mamba
{
try
{
std::ifstream msgs(messages_file);
std::ifstream msgs = open_ifstream(messages_file);
std::stringstream res;
std::copy(std::istreambuf_iterator<char>(msgs),
std::istreambuf_iterator<char>(),
@ -361,7 +362,7 @@ namespace mamba
auto tf = std::make_unique<TemporaryFile>("mamba_bat_", ".bat");
std::ofstream out(tf->path());
std::ofstream out = open_ofstream(tf->path());
std::string silencer = debug_wrapper_scripts ? "" : "@";
@ -404,7 +405,7 @@ namespace mamba
}
#else
auto tf = std::make_unique<TemporaryFile>();
std::ofstream out(tf->path());
std::ofstream out = open_ofstream(tf->path());
std::stringstream hook_quoted;
std::string shebang, dev_arg;
@ -704,7 +705,7 @@ namespace mamba
LOG_INFO << "Unlinking package '" << m_specifier << "'";
LOG_DEBUG << "Use metadata found at '" << json.string() << "'";
std::ifstream json_file(json);
std::ifstream json_file = open_ifstream(json);
nlohmann::json json_record;
json_file >> json_record;
@ -852,7 +853,7 @@ namespace mamba
if (!shebang.empty() && !launcher.empty())
{
replace_all(shebang, path_data.prefix_placeholder, new_prefix);
std::ofstream fo(dst, std::ios::out | std::ios::binary);
std::ofstream fo = open_ofstream(dst, std::ios::out | std::ios::binary);
fo << launcher << shebang << (buffer.c_str() + arc_pos);
fo.close();
}
@ -893,7 +894,7 @@ namespace mamba
#endif
}
std::ofstream fo(dst, std::ios::out | std::ios::binary);
std::ofstream fo = open_ofstream(dst, std::ios::out | std::ios::binary);
fo << buffer;
fo.close();
@ -994,7 +995,7 @@ namespace mamba
std::vector<fs::path> pyc_files;
TemporaryFile all_py_files;
std::ofstream all_py_files_f(all_py_files.path());
std::ofstream all_py_files_f = open_ofstream(all_py_files.path());
for (auto& f : py_files)
{
@ -1088,7 +1089,7 @@ namespace mamba
auto paths_data = read_paths(m_source);
LOG_TRACE << "Opening: " << m_source / "info" / "repodata_record.json";
std::ifstream repodata_f(m_source / "info" / "repodata_record.json");
std::ifstream repodata_f = open_ifstream(m_source / "info" / "repodata_record.json");
repodata_f >> index_json;
std::string f_name = index_json["name"].get<std::string>() + "-"
@ -1183,7 +1184,7 @@ namespace mamba
nlohmann::json link_json;
if (fs::exists(link_json_path))
{
std::ifstream link_json_file(link_json_path);
std::ifstream link_json_file = open_ifstream(link_json_path);
link_json_file >> link_json;
}
@ -1268,7 +1269,7 @@ namespace mamba
LOG_DEBUG << "Finalizing linking";
auto meta = prefix_meta / (f_name + ".json");
LOG_TRACE << "Adding package to prefix metadata at '" << meta.string() << "'";
std::ofstream out_file(meta);
std::ofstream out_file = open_ofstream(meta);
out_file << out_json.dump(4);
return true;

2
libmamba/src/core/package_paths.cpp
View File

@ -113,7 +113,7 @@ namespace mamba
if (fs::exists(paths_json_path))
{
nlohmann::json paths_json;
std::ifstream paths_file(paths_json_path);
std::ifstream paths_file = open_ifstream(paths_json_path);
paths_file >> paths_json;
if (paths_json["paths_version"] != 1)
{

6
libmamba/src/core/prefix_data.cpp
View File

@ -14,9 +14,9 @@
namespace mamba
{
PrefixData::PrefixData(const std::string& prefix_path)
PrefixData::PrefixData(const fs::path& prefix_path)
: m_history(prefix_path)
, m_prefix_path(fs::path(prefix_path))
, m_prefix_path(prefix_path)
{
}
@ -115,7 +115,7 @@ namespace mamba
void PrefixData::load_single_record(const fs::path& path)
{
LOG_INFO << "Loading single package record: " << path;
std::ifstream infile(path);
auto infile = open_ifstream(path);
nlohmann::json j;
infile >> j;
auto prec = PackageInfo(std::move(j));

44
libmamba/src/core/repo.cpp
View File

@ -192,7 +192,7 @@ namespace mamba
return m_repo->nsolvables;
}
const std::string& MRepo::index_file()
const fs::path& MRepo::index_file()
{
return m_json_file;
}
@ -224,34 +224,38 @@ namespace mamba
}
}
bool MRepo::read_file(const std::string& filename)
bool MRepo::read_file(const fs::path& filename)
{
bool is_solv = ends_with(filename, ".solv");
bool is_solv = filename.extension() == ".solv";
std::string filename_wo_extension;
fs::path filename_wo_extension;
if (is_solv)
{
m_solv_file = filename;
filename_wo_extension = filename.substr(0, filename.size() - strlen(".solv"));
m_json_file = filename_wo_extension + ".json";
m_json_file = filename;
m_json_file.replace_extension("json");
}
else
{
m_json_file = filename;
filename_wo_extension = filename.substr(0, filename.size() - strlen(".json"));
m_solv_file = filename_wo_extension + ".solv";
m_solv_file = filename;
m_solv_file.replace_extension("solv");
}
LOG_INFO << "Reading cache files '" << filename_wo_extension << ".*' for repo index '"
<< m_repo->name << "'";
LOG_INFO << "Reading cache files '" << (filename.parent_path() / filename.stem()).string()
<< ".*' for repo index '" << m_repo->name << "'";
if (is_solv)
{
auto lock = LockFile::try_lock(m_solv_file);
#ifdef _WIN32
auto fp = _wfopen(m_solv_file.wstring().c_str(), L"rb");
#else
auto fp = fopen(m_solv_file.c_str(), "rb");
#endif
if (!fp)
{
throw std::runtime_error("Could not open repository file " + filename);
throw std::runtime_error("Could not open repository file " + filename.string());
}
LOG_DEBUG << "Attempt load from solv " << m_solv_file;
@ -318,20 +322,24 @@ namespace mamba
}
auto lock = LockFile::try_lock(m_json_file);
#ifdef _WIN32
auto fp = _wfopen(m_json_file.wstring().c_str(), L"r");
#else
auto fp = fopen(m_json_file.c_str(), "r");
#endif
if (!fp)
{
throw std::runtime_error("Could not open repository file " + m_json_file);
throw std::runtime_error("Could not open repository file " + m_json_file.string());
}
LOG_DEBUG << "Loading JSON file '" << m_json_file << "'";
LOG_DEBUG << "Loading JSON file '" << m_json_file.string() << "'";
int flags = Context::instance().use_only_tar_bz2 ? CONDA_ADD_USE_ONLY_TAR_BZ2 : 0;
int ret = repo_add_conda(m_repo, fp, flags);
if (ret != 0)
{
fclose(fp);
throw std::runtime_error("Could not read JSON repodata file (" + m_json_file + ") "
+ std::string(pool_errstr(m_repo->pool)));
throw std::runtime_error("Could not read JSON repodata file (" + m_json_file.string()
+ ") " + std::string(pool_errstr(m_repo->pool)));
}
// TODO move this to a more structured approach for repodata patching?
@ -355,7 +363,7 @@ namespace mamba
{
Repodata* info;
LOG_INFO << "Writing SOLV file '" << fs::path(m_solv_file).filename().string() << "'";
LOG_INFO << "Writing SOLV file '" << m_solv_file.filename().string() << "'";
info = repo_add_repodata(m_repo, 0); // add new repodata for our meta info
repodata_set_str(info, SOLVID_META, REPOSITORY_TOOLVERSION, mamba_tool_version());
@ -372,7 +380,11 @@ namespace mamba
repodata_internalize(info);
#ifdef _WIN32
auto solv_f = _wfopen(m_solv_file.wstring().c_str(), L"wb");
#else
auto solv_f = fopen(m_solv_file.c_str(), "wb");
#endif
if (!solv_f)
{
LOG_ERROR << "Failed to open .solv file";

31
libmamba/src/core/shell_init.cpp
View File

@ -356,12 +356,12 @@ namespace mamba
if (result.find("# >>> mamba initialize >>>") == result.npos)
{
std::ofstream rc_file(file_path, std::ios::app | std::ios::binary);
std::ofstream rc_file = open_ofstream(file_path, std::ios::app | std::ios::binary);
rc_file << std::endl << conda_init_content;
}
else
{
std::ofstream rc_file(file_path, std::ios::out | std::ios::binary);
std::ofstream rc_file = open_ofstream(file_path, std::ios::out | std::ios::binary);
rc_file << result;
}
return true;
@ -424,7 +424,7 @@ namespace mamba
// Maybe the prefix isn't writable. No big deal, just keep going.
}
std::ofstream mamba_bat_f(root_prefix / "condabin" / "micromamba.bat");
std::ofstream mamba_bat_f = open_ofstream(root_prefix / "condabin" / "micromamba.bat");
std::string mamba_bat_contents(mamba_bat);
replace_all(mamba_bat_contents,
std::string("__MAMBA_INSERT_ROOT_PREFIX__"),
@ -434,7 +434,8 @@ namespace mamba
std::string("@SET \"MAMBA_EXE=" + exe.string() + "\""));
mamba_bat_f << mamba_bat_contents;
std::ofstream _mamba_activate_bat_f(root_prefix / "condabin" / "_mamba_activate.bat");
std::ofstream _mamba_activate_bat_f
= open_ofstream(root_prefix / "condabin" / "_mamba_activate.bat");
_mamba_activate_bat_f << _mamba_activate_bat;
@ -447,10 +448,12 @@ namespace mamba
std::string("@SET \"MAMBA_EXE=" + exe.string() + "\""));
std::ofstream condabin_activate_bat_f(root_prefix / "condabin" / "activate.bat");
std::ofstream condabin_activate_bat_f
= open_ofstream(root_prefix / "condabin" / "activate.bat");
condabin_activate_bat_f << activate_bat_contents;
std::ofstream scripts_activate_bat_f(root_prefix / "Scripts" / "activate.bat");
std::ofstream scripts_activate_bat_f
= open_ofstream(root_prefix / "Scripts" / "activate.bat");
scripts_activate_bat_f << activate_bat_contents;
std::string hook_content = mamba_hook_bat;
@ -458,7 +461,7 @@ namespace mamba
std::string("__MAMBA_INSERT_MAMBA_EXE__"),
std::string("@SET \"MAMBA_EXE=" + exe.string() + "\""));
std::ofstream mamba_hook_bat_f(root_prefix / "condabin" / "mamba_hook.bat");
std::ofstream mamba_hook_bat_f = open_ofstream(root_prefix / "condabin" / "mamba_hook.bat");
mamba_hook_bat_f << hook_content;
}
@ -478,7 +481,7 @@ namespace mamba
{
// Maybe the prefix isn't writable. No big deal, just keep going.
}
std::ofstream sh_file(sh_source_path);
std::ofstream sh_file = open_ofstream(sh_source_path);
sh_file << mamba_sh;
}
if (shell == "xonsh")
@ -493,7 +496,7 @@ namespace mamba
{
// Maybe the prefix isn't writable. No big deal, just keep going.
}
std::ofstream sh_file(sh_source_path);
std::ofstream sh_file = open_ofstream(sh_source_path);
sh_file << mamba_xsh;
}
else if (shell == "cmd.exe")
@ -510,9 +513,9 @@ namespace mamba
{
// Maybe the prefix isn't writable. No big deal, just keep going.
}
std::ofstream mamba_hook_f(root_prefix / "condabin" / "mamba_hook.ps1");
std::ofstream mamba_hook_f = open_ofstream(root_prefix / "condabin" / "mamba_hook.ps1");
mamba_hook_f << mamba_hook_ps1;
std::ofstream mamba_psm1_f(root_prefix / "condabin" / "Mamba.psm1");
std::ofstream mamba_psm1_f = open_ofstream(root_prefix / "condabin" / "Mamba.psm1");
mamba_psm1_f << mamba_psm1;
}
}
@ -578,12 +581,14 @@ namespace mamba
if (!found_mamba_initialize)
{
std::ofstream out(profile_path, std::ios::app | std::ios::binary);
std::ofstream out
= open_ofstream(profile_path, std::ios::app | std::ios::binary);
out << std::endl << conda_init_content;
}
else
{
std::ofstream out(profile_path, std::ios::out | std::ios::binary);
std::ofstream out
= open_ofstream(profile_path, std::ios::out | std::ios::binary);
out << profile_content;
}

8
libmamba/src/core/subdirdata.cpp
View File

@ -32,7 +32,7 @@ namespace decompress
}
struct archive_entry* entry;
std::ofstream out_file(out);
std::ofstream out_file = mamba::open_ofstream(out);
char buff[BLOCKSIZE];
std::size_t buffsize = BLOCKSIZE;
r = archive_read_next_header(a, &entry);
@ -360,7 +360,7 @@ namespace mamba
LOG_DEBUG << "Opening '" << json_file.string() << "'";
path::touch(json_file, true);
std::ofstream final_file(json_file);
std::ofstream final_file = open_ofstream(json_file);
if (!final_file.is_open())
{
@ -376,7 +376,7 @@ namespace mamba
m_progress_bar.set_postfix("Finalizing...");
std::ifstream temp_file(m_temp_file->path());
std::ifstream temp_file = open_ifstream(m_temp_file->path());
std::stringstream temp_json;
temp_json << m_mod_etag.dump();
@ -493,7 +493,7 @@ namespace mamba
return std::string();
};
std::ifstream in_file(file);
std::ifstream in_file = open_ifstream(file);
auto json = extract_subjson(in_file);
nlohmann::json result;
try

2
libmamba/src/core/transaction.cpp
View File

@ -83,7 +83,7 @@ namespace mamba
fs::path index_path = base_path / "info" / "index.json";
nlohmann::json index, solvable_json;
std::ifstream index_file(index_path);
std::ifstream index_file = open_ifstream(index_path);
index_file >> index;
solvable_json = m_package_info.json_record();

9
libmamba/src/core/util.cpp
View File

@ -171,7 +171,7 @@ namespace mamba
try
{
std::ofstream f(final_path);
std::ofstream f = open_ofstream(final_path);
f.close();
success = true;
}
@ -388,7 +388,11 @@ namespace mamba
std::string read_contents(const fs::path& file_path, std::ios::openmode mode)
{
#ifdef _WIN32
std::ifstream in(file_path.wstring(), std::ios::in | mode);
#else
std::ifstream in(file_path, std::ios::in | mode);
#endif
if (in)
{
std::string contents;
@ -814,7 +818,7 @@ namespace mamba
{
// Windows locks are isolated between file descriptor
// We can then test if locked by opening a new one
int fd = _open(path.c_str(), O_RDWR | O_CREAT, 0666);
int fd = _wopen(path.wstring().c_str(), O_RDWR | O_CREAT, 0666);
_lseek(fd, MAMBA_LOCK_POS, SEEK_SET);
char buffer[1];
bool is_locked = _read(fd, buffer, 1) == -1;
@ -1134,5 +1138,4 @@ namespace mamba
throw std::runtime_error("Subprocess call failed. Aborting.");
}
}
} // namespace mamba

89
libmamba/src/core/util_os.cpp
View File

@ -8,6 +8,7 @@
#ifndef _WIN32
#include <unistd.h>
#include <clocale>
#if defined(__APPLE__)
#include <mach-o/dyld.h>
#include <libProc.h>
@ -19,6 +20,7 @@
#include <limits.h>
#endif
#else
#include <atomic>
#include <windows.h>
#include <intrin.h>
#include <tlhelp32.h>
@ -398,4 +400,91 @@ namespace mamba
}
#endif
#ifdef _WIN32
namespace
{
static std::atomic<int> init_console_cp(0);
static std::atomic<int> init_console_output_cp(0);
static std::atomic<bool> init_console_initialized(false);
}
#endif
// init console to make sure UTF8 is properly activated
void init_console()
{
#ifdef _WIN32
init_console_cp = GetConsoleCP();
init_console_output_cp = GetConsoleOutputCP();
init_console_initialized = true;
SetConsoleCP(CP_UTF8);
SetConsoleOutputCP(CP_UTF8);
// Enable buffering to prevent VS from chopping up UTF-8 byte sequences
setvbuf(stdout, nullptr, _IOFBF, 1000);
#else
static const char* const utf8_locales[] = {
"C.UTF-8",
"POSIX.UTF-8",
"en_US.UTF-8",
};
for (const char* utf8_locale : utf8_locales)
{
if (::setlocale(LC_ALL, utf8_locale))
{
::setenv("LC_ALL", utf8_locale, true);
break;
}
}
#endif
}
void reset_console()
{
#ifdef _WIN32
if (init_console_initialized)
{
SetConsoleCP(init_console_cp);
SetConsoleOutputCP(init_console_output_cp);
}
#endif
}
#ifdef _WIN32
std::string to_utf8(const wchar_t* w, size_t s)
{
std::string output;
if (s != 0)
{
assert(s <= INT_MAX);
const int size = WideCharToMultiByte(
CP_UTF8, 0, w, static_cast<int>(s), nullptr, 0, nullptr, nullptr);
if (size <= 0)
{
unsigned long last_error = ::GetLastError();
LOG_ERROR << "Failed to convert string to UTF-8 "
<< std::system_category().message(static_cast<int>(last_error));
throw std::runtime_error("Failed to convert string to UTF-8");
}
output.resize(size);
int res_size = WideCharToMultiByte(CP_UTF8,
0,
w,
static_cast<int>(s),
output.data(),
static_cast<int>(size),
nullptr,
nullptr);
assert(res_size == size);
}
return output;
}
std::string to_utf8(const wchar_t* w)
{
return to_utf8(w, wcslen(w));
}
#endif
}

12
libmamba/src/core/validate.cpp
View File

@ -87,14 +87,14 @@ namespace validate
{
}
std::string sha256sum(const std::string& path)
std::string sha256sum(const fs::path& path)
{
std::array<unsigned char, SHA256_DIGEST_LENGTH> hash;
SHA256_CTX sha256;
SHA256_Init(&sha256);
std::ifstream infile(path, std::ios::binary);
std::ifstream infile = mamba::open_ifstream(path);
constexpr std::size_t BUFSIZE = 32768;
std::vector<char> buffer(BUFSIZE);
@ -113,14 +113,14 @@ namespace validate
return ::mamba::hex_string(hash);
}
std::string md5sum(const std::string& path)
std::string md5sum(const fs::path& path)
{
std::array<unsigned char, MD5_DIGEST_LENGTH> hash;
MD5_CTX md5;
MD5_Init(&md5);
std::ifstream infile(path, std::ios::binary);
auto infile = mamba::open_ifstream(path);
constexpr std::size_t BUFSIZE = 32768;
std::vector<char> buffer(BUFSIZE);
@ -139,12 +139,12 @@ namespace validate
return ::mamba::hex_string(hash);
}
bool sha256(const std::string& path, const std::string& validation)
bool sha256(const fs::path& path, const std::string& validation)
{
return sha256sum(path) == validation;
}
bool md5(const std::string& path, const std::string& validation)
bool md5(const fs::path& path, const std::string& validation)
{
return md5sum(path) == validation;
}

4
libmambapy/src/main.cpp
View File

@ -106,7 +106,7 @@ PYBIND11_MODULE(bindings, m)
.def("solve", &MSolver::solve);
py::class_<History>(m, "History")
.def(py::init<const std::string&>())
.def(py::init<const fs::path&>())
.def("get_requested_specs_map", &History::get_requested_specs_map);
py::class_<MatchSpec>(m, "MatchSpec")
@ -263,7 +263,7 @@ PYBIND11_MODULE(bindings, m)
.def("set_log_level", &Context::set_log_level);
py::class_<PrefixData>(m, "PrefixData")
.def(py::init<const std::string&>())
.def(py::init<const fs::path&>())
.def_readwrite("package_records", &PrefixData::m_package_records)
.def("load", &PrefixData::load)
.def("add_virtual_packages", &PrefixData::add_virtual_packages);

17
mamba/tests/test_all.py
View File

@ -169,3 +169,20 @@ def test_create_files(tmpdir):
output = json.loads(output)
names = {x["name"] for x in output["actions"]["FETCH"]}
assert names == {"a", "b"}
def test_unicode(tmpdir):
uc = "320 áγђß家固êôōçñ한"
output = subprocess.check_output(
["mamba", "create", "-p", str(tmpdir / uc), "--json", "xtensor"]
)
output = json.loads(output)
assert output["prefix"] == str(tmpdir / uc)
import libmambapy
pd = libmambapy.PrefixData(str(tmpdir / uc))
pd.load()
assert len(pd.package_records) > 1
assert "xtl" in pd.package_records
assert "xtensor" in pd.package_records

13
micromamba/src/config.cpp
View File

@ -301,8 +301,7 @@ set_sequence_to_rc(const SequenceAddType& opt)
for (auto& pair : specs)
set_sequence_to_yaml(node, pair.first, pair.second, opt);
std::ofstream rc_file;
rc_file.open(rc_source.string(), std::ofstream::in | std::ofstream::trunc);
std::ofstream rc_file = open_ofstream(rc_source, std::ofstream::in | std::ofstream::trunc);
rc_file << node << std::endl;
config.operation_teardown();
@ -351,7 +350,6 @@ set_config_remove_key_command(CLI::App* subcom)
config.load();
fs::path rc_source = env::expand_user(env::home_directory() / ".condarc");
std::ofstream rc_file;
bool key_removed = false;
if (file_path.configured())
@ -397,7 +395,7 @@ set_config_remove_key_command(CLI::App* subcom)
}
// if the rc file is being modified, it's necessary to rewrite it
rc_file.open(rc_source.string(), std::ofstream::in | std::ofstream::trunc);
std::ofstream rc_file = open_ofstream(rc_source, std::ofstream::in | std::ofstream::trunc);
rc_file << rc_YAML << std::endl;
config.operation_teardown();
@ -434,7 +432,6 @@ set_config_remove_command(CLI::App* subcom)
config.load();
fs::path rc_source = env::expand_user(env::home_directory() / ".condarc");
std::ofstream rc_file;
bool key_removed = false;
std::string remove_vec_key = remove_vec_map.value().front();
std::string remove_vec_value = remove_vec_map.value().at(1);
@ -500,7 +497,7 @@ set_config_remove_command(CLI::App* subcom)
}
// if the rc file is being modified, it's necessary to rewrite it
rc_file.open(rc_source.string(), std::ofstream::in | std::ofstream::trunc);
std::ofstream rc_file = open_ofstream(rc_source, std::ofstream::in | std::ofstream::trunc);
rc_file << rc_YAML << std::endl;
config.operation_teardown();
@ -533,7 +530,6 @@ set_config_set_command(CLI::App* subcom)
config.load();
fs::path rc_source = env::expand_user(env::home_directory() / ".condarc");
std::ofstream rc_file;
if (file_path.configured())
{
@ -562,7 +558,7 @@ set_config_set_command(CLI::App* subcom)
}
// if the rc file is being modified, it's necessary to rewrite it
rc_file.open(rc_source.string(), std::ofstream::in | std::ofstream::trunc);
std::ofstream rc_file = open_ofstream(rc_source, std::ofstream::in | std::ofstream::trunc);
rc_file << rc_YAML << std::endl;
config.operation_teardown();
@ -596,7 +592,6 @@ set_config_get_command(CLI::App* subcom)
config.load();
fs::path rc_source = env::expand_user(env::home_directory() / ".condarc");
std::ofstream rc_file;
bool value_found = false;

2
micromamba/src/constructor.cpp
View File

@ -136,7 +136,7 @@ construct(const fs::path& prefix, bool extract_conda_pkgs, bool extract_tarball)
void
read_binary_from_stdin_and_write_to_file(fs::path& filename)
{
std::ofstream out_stream(filename.string().c_str(), std::ofstream::binary);
std::ofstream out_stream = open_ofstream(filename, std::ofstream::binary);
// Need to reopen stdin as binary
std::freopen(nullptr, "rb", stdin);
if (std::ferror(stdin))

32
micromamba/src/main.cpp
View File

@ -13,6 +13,7 @@
#include "mamba/core/context.hpp"
#include "mamba/core/output.hpp"
#include "mamba/core/thread_utils.hpp"
#include "mamba/core/util_os.hpp"
#include <CLI/CLI.hpp>
@ -23,6 +24,7 @@ using namespace mamba; // NOLINT(build/namespaces)
int
main(int argc, char** argv)
{
init_console();
auto& ctx = Context::instance();
ctx.is_micromamba = true;
@ -31,20 +33,43 @@ main(int argc, char** argv)
CLI::App app{ "Version: " + version() + "\n" };
set_umamba_command(&app);
char** utf8argv;
#ifdef _WIN32
wchar_t** wargv;
wargv = CommandLineToArgvW(GetCommandLineW(), &argc);
std::vector<std::string> utf8Args;
std::vector<char*> utf8CharArgs;
for (int i = 0; i < argc; i++)
{
utf8Args.push_back(to_utf8(wargv[i]));
}
for (int i = 0; i < argc; ++i)
{
utf8CharArgs.push_back(utf8Args[i].data());
}
utf8argv = utf8CharArgs.data();
#else
utf8argv = argv;
#endif
if (argc >= 2 && strcmp(argv[1], "completer") == 0)
{
get_completions(&app, argc, argv);
exit(0);
get_completions(&app, argc, utf8argv);
reset_console();
return 0;
}
try
{
CLI11_PARSE(app, argc, argv);
CLI11_PARSE(app, argc, utf8argv);
}
catch (const std::exception& e)
{
LOG_CRITICAL << e.what();
set_sig_interrupted();
reset_console();
return 1;
}
@ -59,5 +84,6 @@ main(int argc, char** argv)
Console::print(app.get_subcommand("config")->help());
}
reset_console();
return 0;
}

136
micromamba/tests/test_activation.py
View File

@ -80,8 +80,12 @@ if plat == "win":
def write_script(interpreter, lines, path):
fname = os.path.join(path, "script" + suffixes[interpreter])
with open(fname, "w") as fo:
fo.write("\n".join(lines) + "\n")
if plat == "win":
with open(fname, "w", encoding="utf-8-sig") as fo:
fo.write("\n".join(lines) + "\n")
else:
with open(fname, "w") as fo:
fo.write("\n".join(lines) + "\n")
return fname
@ -500,6 +504,134 @@ class TestActivation:
assert find_path_in_str(str(rp / "envs" / "xyz"), res["PATH"])
assert find_path_in_str(str(rp / "envs" / "abc"), res["PATH"])
@pytest.mark.parametrize("interpreter", get_interpreters())
def test_unicode_activation(
self, tmp_path, interpreter, clean_shell_files, new_root_prefix, clean_env
):
if interpreter not in valid_interpreters:
pytest.skip(f"{interpreter} not available")
cwd = os.getcwd()
umamba = get_umamba(cwd=cwd)
s = [f"{umamba} shell init -p {self.root_prefix}"]
stdout, stderr = call_interpreter(s, tmp_path, interpreter)
call = lambda s: call_interpreter(
s, tmp_path, interpreter, interactive=True, env=clean_env
)
if interpreter in ["bash", "zsh"]:
extract_vars = lambda vxs: [f"echo {v}=${v}" for v in vxs]
elif interpreter in ["cmd.exe"]:
extract_vars = lambda vxs: [f"echo {v}=%{v}%" for v in vxs]
elif interpreter in ["powershell"]:
extract_vars = lambda vxs: [f"echo {v}=$Env:{v}" for v in vxs]
elif interpreter in ["fish"]:
extract_vars = lambda vxs: [f"echo {v}=${v}" for v in vxs]
rp = Path(self.root_prefix)
evars = extract_vars(["CONDA_PREFIX", "CONDA_SHLVL", "PATH"])
if interpreter == "cmd.exe":
x = read_windows_registry(regkey)
fp = Path(x[0][1:-1])
assert fp.exists()
if interpreter in ["bash", "zsh", "powershell", "cmd.exe"]:
stdout, stderr = call(evars)
s = [f"{umamba} --help"]
stdout, stderr = call(s)
s = ["micromamba activate"] + evars
stdout, stderr = call(s)
res = TestActivation.to_dict(stdout)
assert "condabin" in res["PATH"]
assert self.root_prefix in res["PATH"]
assert f"CONDA_PREFIX={self.root_prefix}" in stdout.splitlines()
assert f"CONDA_SHLVL=1" in stdout.splitlines()
# throw with non-existent
s = ["micromamba activate nonexistent"]
if not interpreter == "powershell":
with pytest.raises(subprocess.CalledProcessError):
stdout, stderr = call(s)
u1 = "μυρτιὲς"
u2 = "终过鬼门关"
u3 = "some ™∞¢3 spaces §∞©ƒ√≈ç"
s1 = [f"micromamba create -n {u1} xtensor -y -c conda-forge"]
s2 = [f"micromamba create -n {u2} xtensor -y -c conda-forge"]
s3 = [f"micromamba create -n '{u3}' xtensor -y -c conda-forge"]
call(s1)
call(s2)
call(s3)
assert (rp / "envs" / u1 / "conda-meta").is_dir()
assert (rp / "envs" / u2 / "conda-meta").is_dir()
assert (rp / "envs" / u3 / "conda-meta").is_dir()
assert (rp / "envs" / u1 / "conda-meta" / "history").exists()
assert (rp / "envs" / u2 / "conda-meta" / "history").exists()
assert (rp / "envs" / u3 / "conda-meta" / "history").exists()
if plat == "win":
assert (
rp / "envs" / u1 / "Library" / "include" / "xtensor" / "xtensor.hpp"
).exists()
assert (
rp / "envs" / u2 / "Library" / "include" / "xtensor" / "xtensor.hpp"
).exists()
assert (
rp / "envs" / u3 / "Library" / "include" / "xtensor" / "xtensor.hpp"
).exists()
else:
assert (
rp / "envs" / u1 / "include" / "xtensor" / "xtensor.hpp"
).exists()
assert (
rp / "envs" / u2 / "include" / "xtensor" / "xtensor.hpp"
).exists()
assert (
rp / "envs" / u3 / "include" / "xtensor" / "xtensor.hpp"
).exists()
# unicode activation on win: todo
if plat == "win":
return
s = [
f"micromamba activate",
f"micromamba activate {u1}",
f"micromamba activate {u2}",
] + evars
stdout, stderr = call(s)
res = TestActivation.to_dict(stdout)
assert find_path_in_str(str(rp / "condabin"), res["PATH"])
assert not find_path_in_str(str(rp / "bin"), res["PATH"])
assert find_path_in_str(str(rp / "envs" / u2), res["PATH"])
assert not find_path_in_str(str(rp / "envs" / u1), res["PATH"])
s = [
"micromamba activate",
f"micromamba activate {u1}",
f"micromamba activate {u2} --stack",
] + evars
stdout, stderr = call(s)
res = TestActivation.to_dict(stdout)
assert find_path_in_str(str(rp / "condabin"), res["PATH"])
assert not find_path_in_str(str(rp / "bin"), res["PATH"])
assert find_path_in_str(str(rp / "envs" / u1), res["PATH"])
assert find_path_in_str(str(rp / "envs" / u2), res["PATH"])
s = ["micromamba activate", f"micromamba activate '{u3}'",] + evars
stdout, stderr = call(s)
res = TestActivation.to_dict(stdout)
assert find_path_in_str(str(rp / "condabin"), res["PATH"])
assert not find_path_in_str(str(rp / "bin"), res["PATH"])
assert find_path_in_str(str(rp / "envs" / u3), res["PATH"])
@pytest.mark.parametrize("interpreter", get_interpreters())
def test_activate_path(self, tmp_path, interpreter, new_root_prefix):
if interpreter not in valid_interpreters: