folkslinux
/
mamba
mirror of https://github.com/mamba-org/mamba.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

clean up output and download

This commit is contained in:
Wolf Vollprecht 2020-04-15 10:25:51 +02:00
parent 1980fefcf8
commit 987aa00267
6 changed files with 530 additions and 419 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

319
include/fetch.hpp Normal file
View File

@ -0,0 +1,319 @@
#pragma once
#include "thirdparty/indicators/dynamic_progress.hpp"
#include "thirdparty/indicators/progress_bar.hpp"
extern "C" {
#include <stdio.h>
#include <string.h>
/* somewhat unix-specific */
#include <sys/time.h>
#include <unistd.h>
#include <curl/curl.h>
#include <archive.h>
}
#define PREFIX_LENGTH 25
namespace mamba
{
class DownloadTarget
{
public:
static size_t write_callback(char *ptr, size_t size, size_t nmemb, void *self)
{
auto* s = (DownloadTarget*)self;
s->m_file.write(ptr, size * nmemb);
return size * nmemb;
}
DownloadTarget() = default;
DownloadTarget(const std::string& name, const std::string& url, const std::string& filename)
: m_name(name)
{
m_file = std::ofstream(filename, std::ios::binary);
m_target = curl_easy_init();
curl_easy_setopt(m_target, CURLOPT_URL, url.c_str());
curl_easy_setopt(m_target, CURLOPT_HEADERFUNCTION, &DownloadTarget::header_callback);
curl_easy_setopt(m_target, CURLOPT_HEADERDATA, this);
curl_easy_setopt(m_target, CURLOPT_WRITEFUNCTION, &DownloadTarget::write_callback);
curl_easy_setopt(m_target, CURLOPT_WRITEDATA, this);
m_headers = nullptr;
if (ends_with(url, ".json"))
{
curl_easy_setopt(m_target, CURLOPT_ACCEPT_ENCODING, "gzip, deflate, compress, identity");
m_headers = curl_slist_append(m_headers, "Content-Type: application/json");
}
curl_easy_setopt(m_target, CURLOPT_HTTPHEADER, m_headers);
curl_easy_setopt(m_target, CURLOPT_VERBOSE, Context::instance().verbosity != 0);
}
void set_mod_etag_headers(const nlohmann::json& mod_etag)
{
auto to_header = [](const std::string& key, const std::string& value) {
return std::string(key + ": " + value);
};
if (mod_etag.find("_etag") != mod_etag.end())
{
m_headers = curl_slist_append(m_headers, to_header("If-None-Match", mod_etag["_etag"]).c_str());
}
if (mod_etag.find("_mod") != mod_etag.end())
{
m_headers = curl_slist_append(m_headers, to_header("If-Modified-Since", mod_etag["_mod"]).c_str());
}
}
void set_progress_callback(int (*cb)(void*, curl_off_t, curl_off_t, curl_off_t, curl_off_t), void* data)
{
curl_easy_setopt(m_target, CURLOPT_XFERINFOFUNCTION, cb);
curl_easy_setopt(m_target, CURLOPT_XFERINFODATA, data);
curl_easy_setopt(m_target, CURLOPT_NOPROGRESS, 0L);
}
static size_t header_callback(char *buffer, size_t size, size_t nitems, void *self)
{
auto* s = (DownloadTarget*)self;
std::string_view header(buffer, size * nitems);
auto colon_idx = header.find(':');
if (colon_idx != std::string_view::npos)
{
std::string_view key, value;
key = header.substr(0, colon_idx);
colon_idx++;
// remove spaces
while (std::isspace(header[colon_idx]))
{
++colon_idx;
}
// remove \r\n header ending
value = header.substr(colon_idx, header.size() - colon_idx - 2);
if (key == "ETag")
{
s->etag = value;
}
else if (key == "Cache-Control")
{
s->cache_control = value;
}
else if (key == "Last-Modified")
{
s->mod = value;
}
}
return nitems * size;
}
const std::string& name() const
{
return m_name;
}
bool perform()
{
CURLcode res = curl_easy_perform(m_target);
if (res != CURLE_OK)
{
throw std::runtime_error(curl_easy_strerror(res));
}
if (m_finalize_callback)
{
return m_finalize_callback();
}
else return true;
}
CURL* handle()
{
return m_target;
}
curl_off_t get_speed()
{
curl_off_t speed;
CURLcode res = curl_easy_getinfo(m_target, CURLINFO_SPEED_DOWNLOAD_T, &speed);
if (res == CURLE_OK)
{
return speed;
}
return 0;
}
template <class C>
void set_finalize_callback(int (C::*cb)(), C* data)
{
m_finalize_callback = std::bind(cb, data);
}
bool finalize()
{
m_file.flush();
char* effective_url = nullptr;
curl_easy_getinfo(m_target, CURLINFO_RESPONSE_CODE, &http_status);
curl_easy_getinfo(m_target, CURLINFO_EFFECTIVE_URL, &effective_url);
LOG(INFO) << "Transfer finalized, status: " << http_status << " @ " << effective_url;
final_url = effective_url;
if (m_finalize_callback)
{
return m_finalize_callback();
}
return true;
}
~DownloadTarget()
{
curl_easy_cleanup(m_target);
curl_slist_free_all(m_headers);
}
int http_status;
std::string final_url;
std::string etag, mod, cache_control;
private:
std::function<int()> m_finalize_callback;
std::string m_name;
CURL* m_target;
curl_slist* m_headers;
std::ofstream m_file;
};
class MultiDownloadTarget
{
public:
MultiDownloadTarget()
{
m_handle = curl_multi_init();
}
~MultiDownloadTarget()
{
curl_multi_cleanup(m_handle);
}
void add(std::unique_ptr<DownloadTarget>& target)
{
if (!target) return;
CURLMcode code = curl_multi_add_handle(m_handle, target->handle());
if(code != CURLM_CALL_MULTI_PERFORM)
{
if(code != CURLM_OK)
{
throw std::runtime_error(curl_multi_strerror(code));
}
}
// subdirdata.set_progress_bar(idx, &m_multi_progress);
m_targets.push_back(target.get());
}
bool check_msgs()
{
int msgs_in_queue;
CURLMsg *msg;
while ((msg = curl_multi_info_read(m_handle, &msgs_in_queue))) {
if (msg->data.result != CURLE_OK) {
char* effective_url = nullptr;
curl_easy_getinfo(msg->easy_handle, CURLINFO_EFFECTIVE_URL, &effective_url);
std::stringstream err;
err << "Download error (" << msg->data.result << ") " <<
curl_easy_strerror(msg->data.result) << "[" << effective_url << "]";
throw std::runtime_error(err.str());
}
if (msg->msg != CURLMSG_DONE) {
// We are only interested in messages about finished transfers
continue;
}
DownloadTarget* current_target = nullptr;
for (const auto& target : m_targets)
{
if (target->handle() == msg->easy_handle)
{
current_target = target;
break;
}
}
if (!current_target)
{
throw std::runtime_error("Could not find target associated with multi request");
}
// flush file & finalize transfer
current_target->finalize();
}
return true;
}
bool download(bool failfast)
{
LOG(INFO) << "Starting to download targets";
int still_running, repeats = 0;
const long max_wait_msecs = 400;
do
{
CURLMcode code = curl_multi_perform(m_handle, &still_running);
if(code != CURLM_OK)
{
throw std::runtime_error(curl_multi_strerror(code));
}
check_msgs();
int numfds;
code = curl_multi_wait(m_handle, NULL, 0, max_wait_msecs, &numfds);
if (code != CURLM_OK)
{
throw std::runtime_error(curl_multi_strerror(code));
}
if(!numfds)
{
repeats++; // count number of repeated zero numfds
if(repeats > 1)
{
// wait 100 ms
#ifdef _WIN32
Sleep(100)
#else
// Portable sleep for platforms other than Windows.
struct timeval wait = { 0, 100 * 1000 };
(void) select(0, NULL, NULL, NULL, &wait);
#endif
}
}
else
{
repeats = 0;
}
} while (still_running);
return true;
}
std::vector<DownloadTarget*> m_targets;
CURLM* m_handle;
};
}

123
include/output.hpp Normal file
View File

@ -0,0 +1,123 @@
#include <iostream>
#include <string>
#include "context.hpp"
namespace mamba
{
class NullBuffer : public std::streambuf
{
public:
int overflow(int c) { return c; }
};
class NullStream : public std::ostream
{
public:
NullStream()
: std::ostream(&m_sb)
{
}
private:
NullBuffer m_sb;
};
class Output
{
public:
static std::ostream& print()
{
if (!Context::instance().quiet)
{
return std::cout;
}
else
{
return Output::instance().null_stream;
}
}
static void print(const std::string_view& str)
{
if (!Context::instance().quiet)
{
std::cout << str << std::endl;
}
}
struct ProgressProxy
{
indicators::ProgressBar* p_bar;
void set_progress(std::size_t p)
{
p_bar->set_progress(p);
Output::instance().print_progress();
}
template <class T>
void set_option(T&& option)
{
p_bar->set_option(std::forward<T>(option));
Output::instance().print_progress();
}
void mark_as_completed()
{
p_bar->mark_as_completed();
Output::instance().print_progress();
}
};
ProgressProxy add_progress_bar(const std::string& name)
{
std::string prefix = name;
prefix.resize(PREFIX_LENGTH - 1, ' ');
m_progress_bars.push_back(std::make_unique<indicators::ProgressBar>(
indicators::option::BarWidth{15},
indicators::option::ForegroundColor{indicators::Color::unspecified},
indicators::option::ShowElapsedTime{true},
indicators::option::ShowRemainingTime{false},
indicators::option::MaxPostfixTextLen{36},
indicators::option::PrefixText(prefix)
));
m_progress_bars[m_progress_bars.size() - 1].get()->multi_progress_mode_ = true;
return ProgressProxy{m_progress_bars[m_progress_bars.size() - 1].get()};
}
void init_multi_progress()
{
m_progress_bars.clear();
m_progress_started = false;
}
void print_progress()
{
auto count_up = m_progress_bars.size();
if (m_progress_started)
{
std::cout << "\x1b[" << count_up << "A";
}
for (auto& bar : m_progress_bars)
{
bar->print_progress(true);
std::cout << "\n";
}
m_progress_started = true;
}
static Output& instance()
{
static Output out;
return out;
}
NullStream null_stream;
private:
Output() {}
std::vector<std::unique_ptr<indicators::ProgressBar>> m_progress_bars;
bool m_progress_started = false;
};
}

6
include/py_interface.cpp
View File

@ -1,5 +1,3 @@
// #include "solver.hpp"
#include "thirdparty/filesystem.hpp"
namespace fs = ghc::filesystem;
@ -68,7 +66,9 @@ PYBIND11_MODULE(mamba_api, m) {
py::class_<MultiDownloadTarget>(m, "DownloadTargetList")
.def(py::init<>())
.def("add", &MultiDownloadTarget::add)
.def("add", [](MultiDownloadTarget& self, MSubdirData& sub) -> void {
self.add(sub.target());
})
.def("download", &MultiDownloadTarget::download)
;

413
include/subdirdata.hpp
View File

@ -8,34 +8,11 @@
#include "context.hpp"
#include "util.hpp"
extern "C" {
#include <stdio.h>
#include <string.h>
/* somewhat unix-specific */
#include <sys/time.h>
#include <unistd.h>
#include <curl/curl.h>
#include <archive.h>
}
#include "fetch.hpp"
#include "output.hpp"
namespace fs = ghc::filesystem;
#define PREFIX_LENGTH 25
void to_human_readable_filesize(std::ostream& o, double bytes, std::size_t precision = 0)
{
const char* sizes[] = { " B", "KB", "MB", "GB", "TB" };
int order = 0;
while (bytes >= 1024 && order < (5 - 1)) {
order++;
bytes = bytes / 1024;
}
o << std::fixed << std::setprecision(precision) << bytes << sizes[order];
}
namespace decompress
{
bool raw(const std::string& in, const std::string& out)
@ -85,139 +62,6 @@ namespace decompress
namespace mamba
{
class DownloadTarget
{
public:
static size_t write_callback(char *ptr, size_t size, size_t nmemb, void *self)
{
auto* s = (DownloadTarget*)self;
s->m_file.write(ptr, size * nmemb);
return size * nmemb;
}
DownloadTarget() = default;
DownloadTarget(const std::string& url, const std::string& filename)
{
m_file = std::ofstream(filename, std::ios::binary);
m_target = curl_easy_init();
curl_easy_setopt(m_target, CURLOPT_URL, url.c_str());
curl_easy_setopt(m_target, CURLOPT_HEADERFUNCTION, &DownloadTarget::header_callback);
curl_easy_setopt(m_target, CURLOPT_HEADERDATA, this);
curl_easy_setopt(m_target, CURLOPT_WRITEFUNCTION, &DownloadTarget::write_callback);
curl_easy_setopt(m_target, CURLOPT_WRITEDATA, this);
m_headers = nullptr;
if (ends_with(url, ".json"))
{
curl_easy_setopt(m_target, CURLOPT_ACCEPT_ENCODING, "gzip, deflate, compress, identity");
m_headers = curl_slist_append(m_headers, "Content-Type: application/json");
}
curl_easy_setopt(m_target, CURLOPT_HTTPHEADER, m_headers);
curl_easy_setopt(m_target, CURLOPT_VERBOSE, Context::instance().verbosity != 0);
}
void set_mod_etag_headers(const nlohmann::json& mod_etag)
{
auto to_header = [](const std::string& key, const std::string& value) {
return std::string(key + ": " + value);
};
if (mod_etag.find("_etag") != mod_etag.end())
{
m_headers = curl_slist_append(m_headers, to_header("If-None-Match", mod_etag["_etag"]).c_str());
}
if (mod_etag.find("_mod") != mod_etag.end())
{
m_headers = curl_slist_append(m_headers, to_header("If-Modified-Since", mod_etag["_mod"]).c_str());
}
}
void set_progress_callback(int (*cb)(void*, curl_off_t, curl_off_t, curl_off_t, curl_off_t), void* data)
{
curl_easy_setopt(m_target, CURLOPT_XFERINFOFUNCTION, cb);
curl_easy_setopt(m_target, CURLOPT_XFERINFODATA, data);
curl_easy_setopt(m_target, CURLOPT_NOPROGRESS, 0L);
}
static size_t header_callback(char *buffer, size_t size, size_t nitems, void *self)
{
auto* s = (DownloadTarget*)self;
std::string_view header(buffer, size * nitems);
auto colon_idx = header.find(':');
if (colon_idx != std::string_view::npos)
{
std::string_view key, value;
key = header.substr(0, colon_idx);
colon_idx++;
// remove spaces
while (std::isspace(header[colon_idx]))
{
++colon_idx;
}
// remove \r\n header ending
value = header.substr(colon_idx, header.size() - colon_idx - 2);
if (key == "ETag")
{
s->etag = value;
}
else if (key == "Cache-Control")
{
s->cache_control = value;
}
else if (key == "Last-Modified")
{
s->mod = value;
}
}
return nitems * size;
}
bool perform()
{
return curl_easy_perform(m_target);
}
CURL* handle()
{
return m_target;
}
curl_off_t get_speed()
{
curl_off_t speed;
CURLcode res = curl_easy_getinfo(m_target, CURLINFO_SPEED_DOWNLOAD_T, &speed);
if (!res)
{
return speed;
}
return 0;
}
bool finalize()
{
m_file.flush();
}
~DownloadTarget()
{
curl_easy_cleanup(m_target);
curl_slist_free_all(m_headers);
}
std::string etag, mod, cache_control;
CURL* m_target;
curl_slist* m_headers;
std::ofstream m_file;
};
class MSubdirData
{
public:
@ -283,7 +127,10 @@ namespace mamba
{
// cache valid!
LOG(INFO) << "Using cache " << m_url << " age in seconds: " << cache_age << " / " << max_age;
OUTPUT(std::left << std::setw(PREFIX_LENGTH) << m_name << "Using cache");
std::string prefix = m_name;
prefix.resize(PREFIX_LENGTH - 1, ' ');
Output::print() << prefix << "Using cache\n";
m_loaded = true;
m_json_cache_valid = true;
@ -336,18 +183,10 @@ namespace mamba
return m_name;
}
void set_progress_bar(std::size_t idx, indicators::DynamicProgress<indicators::ProgressBar>* ptr)
int finalize_transfer()
{
m_multi_progress_idx = idx;
p_multi_progress = ptr;
}
int finalize_transfer(int status)
{
auto& mp = *(p_multi_progress);
LOG(WARNING) << "HTTP response code: " << status;
if (status == 304)
LOG(WARNING) << "HTTP response code: " << m_target->http_status;
if (m_target->http_status == 304)
{
// cache still valid
double cache_age = check_cache(m_json_fn);
@ -361,12 +200,9 @@ namespace mamba
m_solv_cache_valid = true;
}
if (!Context::instance().quiet)
{
mp[m_multi_progress_idx].set_option(indicators::option::PostfixText{"No change"});
mp[m_multi_progress_idx].set_progress(100);
mp[m_multi_progress_idx].mark_as_completed();
}
m_progress_bar.set_option(indicators::option::PostfixText{"No change"});
m_progress_bar.set_progress(100);
m_progress_bar.mark_as_completed();
m_json_cache_valid = true;
m_loaded = true;
@ -393,21 +229,16 @@ namespace mamba
if (ends_with(m_url, ".bz2"))
{
if (!Context::instance().quiet)
{
mp[m_multi_progress_idx].set_option(indicators::option::PostfixText{"Decomp..."});
}
m_progress_bar.set_option(indicators::option::PostfixText{"Decomp..."});
m_temp_name = decompress();
}
if (!Context::instance().quiet)
{
mp[m_multi_progress_idx].set_option(indicators::option::PostfixText{"Finalizing..."});
}
m_progress_bar.set_option(indicators::option::PostfixText{"Finalizing..."});
std::ifstream temp_file(m_temp_name);
std::stringstream temp_json;
temp_json << prepend_header.dump();
// replace `}` with `,`
temp_json.seekp(-1, temp_json.cur); temp_json << ',';
final_file << temp_json.str();
@ -418,19 +249,15 @@ namespace mamba
std::ostreambuf_iterator<char>(final_file)
);
if (!Context::instance().quiet)
{
mp[m_multi_progress_idx].set_option(indicators::option::PostfixText{"Done"});
mp[m_multi_progress_idx].set_progress(100);
mp[m_multi_progress_idx].mark_as_completed();
}
m_progress_bar.set_option(indicators::option::PostfixText{"Done"});
m_progress_bar.set_progress(100);
m_progress_bar.mark_as_completed();
m_json_cache_valid = true;
m_loaded = true;
return 0;
}
private:
std::string decompress()
@ -453,23 +280,21 @@ namespace mamba
return 0;
}
auto& mp = (*(s->p_multi_progress));
if (!s->m_download_complete && total_to_download != 0)
{
double perc = double(now_downloaded) / double(total_to_download);
// std::stringstream postfix;
// to_human_readable_filesize(postfix, now_downloaded);
// postfix << " / ";
// to_human_readable_filesize(postfix, total_to_download);
// postfix << " (";
// to_human_readable_filesize(postfix, s->target()->get_speed(), 2);
// postfix << "/s)";
// mp[s->m_multi_progress_idx].set_option(indicators::option::PostfixText{postfix.str()});
mp[s->m_multi_progress_idx].set_progress(perc * 100.);
std::stringstream postfix;
to_human_readable_filesize(postfix, now_downloaded);
postfix << " / ";
to_human_readable_filesize(postfix, total_to_download);
postfix << " (";
to_human_readable_filesize(postfix, s->target()->get_speed(), 2);
postfix << "/s)";
s->m_progress_bar.set_option(indicators::option::PostfixText{postfix.str()});
s->m_progress_bar.set_progress(perc * 100.);
if (std::ceil(perc * 100.) >= 100)
{
mp[s->m_multi_progress_idx].mark_as_completed();
s->m_progress_bar.mark_as_completed();
s->m_download_complete = true;
}
}
@ -480,16 +305,18 @@ namespace mamba
postfix << " / ?? (";
to_human_readable_filesize(postfix, s->target()->get_speed(), 2);
postfix << "/s)";
mp[s->m_multi_progress_idx].set_option(indicators::option::PostfixText{postfix.str()});
s->m_progress_bar.set_option(indicators::option::PostfixText{postfix.str()});
}
return 0;
}
void create_target(nlohmann::json& mod_etag /*, Handle& handle*/)
void create_target(nlohmann::json& mod_etag)
{
m_temp_name = std::tmpnam(nullptr);
m_target = std::make_unique<DownloadTarget>(m_url, m_temp_name);
m_progress_bar = Output::instance().add_progress_bar(m_name);
m_target = std::make_unique<DownloadTarget>(m_name, m_url, m_temp_name);
m_target->set_progress_callback(&MSubdirData::progress_callback, this);
m_target->set_finalize_callback(&MSubdirData::finalize_transfer, this);
m_target->set_mod_etag_headers(mod_etag);
}
@ -563,8 +390,7 @@ namespace mamba
std::ofstream out_file;
indicators::DynamicProgress<indicators::ProgressBar>* p_multi_progress;
std::size_t m_multi_progress_idx;
Output::ProgressProxy m_progress_bar;
bool m_loaded, m_download_complete;
std::string m_url;
@ -573,175 +399,4 @@ namespace mamba
std::string m_solv_fn;
std::string m_temp_name;
};
class MultiDownloadTarget
{
public:
MultiDownloadTarget()
{
m_handle = curl_multi_init();
}
~MultiDownloadTarget()
{
curl_multi_cleanup(m_handle);
}
void add(MSubdirData& subdirdata)
{
if (subdirdata.loaded())
{
LOG(WARNING) << "Adding cached or previously loaded subdirdata, ignoring";
return;
}
if (subdirdata.target() == nullptr)
{
LOG(WARNING) << "Subdirdata load not called, ignoring";
return;
}
CURLMcode code = curl_multi_add_handle(m_handle, subdirdata.target()->handle());
if(code != CURLM_CALL_MULTI_PERFORM)
{
if(code != CURLM_OK)
{
throw std::runtime_error(curl_multi_strerror(code));
}
}
if (subdirdata.target() != nullptr)
{
std::string prefix = subdirdata.name();
if (prefix.size() < (PREFIX_LENGTH - 1)) {
prefix += std::string(PREFIX_LENGTH - prefix.size(), ' ');
}
else if (prefix.size() > (PREFIX_LENGTH - 1))
{
prefix = prefix.substr(0, (PREFIX_LENGTH - 1)) + std::string(" ");
}
m_progress_bars.push_back(std::make_unique<indicators::ProgressBar>(
indicators::option::BarWidth{15},
indicators::option::ForegroundColor{indicators::Color::unspecified},
indicators::option::ShowElapsedTime{true},
indicators::option::ShowRemainingTime{false},
indicators::option::MaxPostfixTextLen{36},
indicators::option::PrefixText(prefix)
));
auto idx = m_multi_progress.push_back(*m_progress_bars[m_progress_bars.size() - 1].get());
subdirdata.set_progress_bar(idx, &m_multi_progress);
m_subdir_data.push_back(&subdirdata);
}
}
bool check_msgs()
{
int msgs_in_queue;
CURLMsg *msg;
while ((msg = curl_multi_info_read(m_handle, &msgs_in_queue))) {
if (msg->data.result != CURLE_OK) {
char* effective_url = nullptr;
curl_easy_getinfo(msg->easy_handle,
CURLINFO_EFFECTIVE_URL,
&effective_url);
std::stringstream err;
err << "Download error (" << msg->data.result << ") " <<
curl_easy_strerror(msg->data.result) << "[" << effective_url << "]";
throw std::runtime_error(err.str());
}
if (msg->msg != CURLMSG_DONE) {
// We are only interested in messages about finished transfers
continue;
}
MSubdirData* subdir_data = nullptr;
for (const auto& sd : m_subdir_data)
{
if (sd->target()->handle() == msg->easy_handle)
{
subdir_data = sd;
break;
}
}
subdir_data->target()->finalize();
if (!subdir_data)
{
throw std::runtime_error("transfer failed");
}
int response_code;
char* effective_url = nullptr;
curl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &response_code);
curl_easy_getinfo(msg->easy_handle, CURLINFO_EFFECTIVE_URL, &effective_url);
LOG(INFO) << "response_code " << response_code << " @ " << effective_url;
subdir_data->finalize_transfer(response_code);
}
return true;
}
bool download(bool failfast)
{
m_multi_progress.set_option(indicators::option::HideBarWhenComplete{false});
for (auto& bar : m_progress_bars)
{
bar->set_progress(0);
}
LOG(INFO) << "Starting to download targets";
int still_running, repeats = 0;
const long max_wait_msecs = 400;
int i = 0;
do
{
CURLMcode code = curl_multi_perform(m_handle, &still_running);
if(code != CURLM_OK)
{
throw std::runtime_error(curl_multi_strerror(code));
}
check_msgs();
int numfds;
code = curl_multi_wait(m_handle, NULL, 0, max_wait_msecs, &numfds);
if (code != CURLM_OK)
{
throw std::runtime_error(curl_multi_strerror(code));
}
if(!numfds)
{
repeats++; // count number of repeated zero numfds
if(repeats > 1)
{
// wait 100 ms
#ifdef _WIN32
Sleep(100)
#else
// Portable sleep for platforms other than Windows.
struct timeval wait = { 0, 100 * 1000 };
(void) select(0, NULL, NULL, NULL, &wait);
#endif
}
}
else
{
repeats = 0;
}
} while (still_running);
return true;
}
std::vector<std::unique_ptr<indicators::ProgressBar>> m_progress_bars;
indicators::DynamicProgress<indicators::ProgressBar> m_multi_progress;
std::vector<MSubdirData*> m_subdir_data;
CURLM* m_handle;
};
}

72
include/thirdparty/indicators/progress_bar.hpp vendored
View File

@ -158,38 +158,6 @@ public:
print_progress();
}
private:
template <details::ProgressBarOption id>
auto get_value() -> decltype((details::get_value<id>(std::declval<Settings &>()).value)) {
return details::get_value<id>(settings_).value;
}
template <details::ProgressBarOption id>
auto get_value() const
-> decltype((details::get_value<id>(std::declval<const Settings &>()).value)) {
return details::get_value<id>(settings_).value;
}
size_t progress_{0};
Settings settings_;
std::chrono::nanoseconds elapsed_;
std::chrono::time_point<std::chrono::high_resolution_clock> start_time_point_;
std::mutex mutex_;
template <typename Indicator, size_t count> friend class MultiProgress;
template <typename Indicator> friend class DynamicProgress;
std::atomic<bool> multi_progress_mode_{false};
void save_start_time() {
auto &show_elapsed_time = get_value<details::ProgressBarOption::show_elapsed_time>();
auto &saved_start_time = get_value<details::ProgressBarOption::saved_start_time>();
auto &show_remaining_time = get_value<details::ProgressBarOption::show_remaining_time>();
if ((show_elapsed_time || show_remaining_time) && !saved_start_time) {
start_time_point_ = std::chrono::high_resolution_clock::now();
saved_start_time = true;
}
}
void print_progress(bool from_multi_progress = false) {
std::lock_guard<std::mutex> lock{mutex_};
if (multi_progress_mode_ && !from_multi_progress) {
@ -200,7 +168,12 @@ private:
}
auto now = std::chrono::high_resolution_clock::now();
if (!get_value<details::ProgressBarOption::completed>())
elapsed_ = std::chrono::duration_cast<std::chrono::nanoseconds>(now - start_time_point_);
{
if (get_value<details::ProgressBarOption::saved_start_time>())
elapsed_ = std::chrono::duration_cast<std::chrono::nanoseconds>(now - start_time_point_);
else
elapsed_ = std::chrono::nanoseconds(0);
}
// std::cout << termcolor::bold;
if (get_value<details::ProgressBarOption::foreground_color>() != Color::unspecified)
@ -260,6 +233,39 @@ private:
!from_multi_progress) // Don't std::endl if calling from MultiProgress
std::cout << termcolor::reset << std::endl;
}
std::atomic<bool> multi_progress_mode_{false};
private:
template <details::ProgressBarOption id>
auto get_value() -> decltype((details::get_value<id>(std::declval<Settings &>()).value)) {
return details::get_value<id>(settings_).value;
}
template <details::ProgressBarOption id>
auto get_value() const
-> decltype((details::get_value<id>(std::declval<const Settings &>()).value)) {
return details::get_value<id>(settings_).value;
}
size_t progress_{0};
Settings settings_;
std::chrono::nanoseconds elapsed_;
std::chrono::time_point<std::chrono::high_resolution_clock> start_time_point_;
std::mutex mutex_;
template <typename Indicator, size_t count> friend class MultiProgress;
template <typename Indicator> friend class DynamicProgress;
void save_start_time() {
auto &show_elapsed_time = get_value<details::ProgressBarOption::show_elapsed_time>();
auto &saved_start_time = get_value<details::ProgressBarOption::saved_start_time>();
auto &show_remaining_time = get_value<details::ProgressBarOption::show_remaining_time>();
if ((show_elapsed_time || show_remaining_time) && !saved_start_time) {
start_time_point_ = std::chrono::high_resolution_clock::now();
saved_start_time = true;
}
}
};
} // namespace indicators

16
include/util.hpp
View File

@ -3,10 +3,7 @@
#include <stdexcept>
#include <string_view>
#include <iostream>
#define OUTPUT(x) \
if (!Context::instance().quiet) \
std::cout << x << "\n";
#include <iomanip>
class mamba_error
: public std::runtime_error
@ -23,4 +20,15 @@ static bool ends_with(const std::string_view& str, const std::string_view& suffi
static bool starts_with(const std::string_view& str, const std::string_view& prefix)
{
return str.size() >= prefix.size() && 0 == str.compare(0, prefix.size(), prefix);
}
void to_human_readable_filesize(std::ostream& o, double bytes, std::size_t precision = 0)
{
const char* sizes[] = { " B", "KB", "MB", "GB", "TB" };
int order = 0;
while (bytes >= 1024 && order < (5 - 1)) {
order++;
bytes = bytes / 1024;
}
o << std::fixed << std::setprecision(precision) << bytes << sizes[order];
}