diff --git a/docs/source/python_api.rst b/docs/source/python_api.rst index ce2677bbd..86f25e46f 100644 --- a/docs/source/python_api.rst +++ b/docs/source/python_api.rst @@ -56,7 +56,7 @@ Here is an example usage of the mamba_api: index.append((sd, channel)) dlist.add(sd) - is_downloaded = dlist.download(True) + is_downloaded = dlist.download(mamba_api.MAMBA_DOWNLOAD_FAILFAST) if not is_downloaded: raise RuntimeError("Error downloading repodata.") diff --git a/libmamba/include/mamba/core/fetch.hpp b/libmamba/include/mamba/core/fetch.hpp index 4c031e383..141fc351e 100644 --- a/libmamba/include/mamba/core/fetch.hpp +++ b/libmamba/include/mamba/core/fetch.hpp @@ -17,7 +17,9 @@ extern "C" #include #include "nlohmann/json.hpp" + #include "output.hpp" +#include "progress_bar.hpp" #include "validate.hpp" namespace mamba @@ -43,6 +45,7 @@ namespace mamba void set_expected_size(std::size_t size); const std::string& name() const; + std::size_t expected_size() const; void init_curl_target(const std::string& url); @@ -74,6 +77,9 @@ namespace mamba bool can_retry(); CURL* retry(); + std::chrono::steady_clock::time_point progress_throttle_time() const; + void set_progress_throttle_time(const std::chrono::steady_clock::time_point& time); + CURLcode result; bool failed = false; int http_status = 10000; @@ -108,6 +114,9 @@ namespace mamba std::ofstream m_file; static void init_curl_handle(CURL* handle, const std::string& url); + std::function download_repr(); + + std::chrono::steady_clock::time_point m_progress_throttle_time; }; class MultiDownloadTarget @@ -118,7 +127,7 @@ namespace mamba void add(DownloadTarget* target); bool check_msgs(bool failfast); - bool download(bool failfast); + bool download(int options); private: std::vector m_targets; @@ -126,6 +135,8 @@ namespace mamba CURLM* m_handle; }; + const int MAMBA_DOWNLOAD_FAILFAST = 1 << 0; + const int MAMBA_DOWNLOAD_SORT = 1 << 1; } // namespace mamba #endif // MAMBA_FETCH_HPP diff --git a/libmamba/include/mamba/core/output.hpp b/libmamba/include/mamba/core/output.hpp index 4712e99c9..76db7ad5b 100644 --- a/libmamba/include/mamba/core/output.hpp +++ b/libmamba/include/mamba/core/output.hpp @@ -46,97 +46,6 @@ #define PREFIX_LENGTH 25 -namespace cursor -{ - class CursorMovementTriple - { - public: - CursorMovementTriple(const char* esc, int n, const char* mod) - : m_esc(esc) - , m_mod(mod) - , m_n(n) - { - } - - const char* m_esc; - const char* m_mod; - int m_n; - }; - - inline std::ostream& operator<<(std::ostream& o, const CursorMovementTriple& m) - { - o << m.m_esc << m.m_n << m.m_mod; - return o; - } - - class CursorMod - { - public: - CursorMod(const char* mod) - : m_mod(mod) - { - } - - std::ostream& operator<<(std::ostream& o) - { - o << m_mod; - return o; - } - - const char* m_mod; - }; - - inline auto up(int n) - { - return CursorMovementTriple("\x1b[", n, "A"); - } - - inline auto down(int n) - { - return CursorMovementTriple("\x1b[", n, "B"); - } - - inline auto forward(int n) - { - return CursorMovementTriple("\x1b[", n, "C"); - } - - inline auto back(int n) - { - return CursorMovementTriple("\x1b[", n, "D"); - } - - inline auto next_line(int n) - { - return CursorMovementTriple("\x1b[", n, "E"); - } - - inline auto prev_line(int n) - { - return CursorMovementTriple("\x1b[", n, "F"); - } - - inline auto horizontal_abs(int n) - { - return CursorMovementTriple("\x1b[", n, "G"); - } - - inline auto erase_line(int n = 0) - { - return CursorMovementTriple("\x1b[", n, "K"); - } - - inline auto show(int n) - { - return CursorMod("\x1b[?25h"); - } - - inline auto hide(int n) - { - return CursorMod("\x1b[?25l"); - } -} // namespace cursor - namespace mamba { std::string cut_repo_name(const std::string& reponame); @@ -204,11 +113,6 @@ namespace mamba std::ostringstream table_like(const std::vector& data, std::size_t max_width); } // namespace printers - // The next two functions / classes were ported from the awesome indicators - // library by p-ranav (MIT License) https://github.com/p-ranav/indicators - std::ostream& write_duration(std::ostream& os, std::chrono::nanoseconds ns); - int get_console_width(); - // Todo: replace public inheritance with // private one + using directives class ConsoleStream : public std::stringstream @@ -236,7 +140,10 @@ namespace mamba std::istream& input_stream = std::cin); ProgressProxy add_progress_bar(const std::string& name, size_t expected_total = 0); - void init_multi_progress(ProgressBarMode mode = ProgressBarMode::multi); + void clear_progress_bars(); + ProgressBarManager& init_progress_bar_manager(ProgressBarMode mode + = ProgressBarMode::multi); + ProgressBarManager& progress_bar_manager(); static std::string hide_secrets(const std::string_view& str); @@ -247,30 +154,26 @@ namespace mamba void json_down(const std::string& key); void json_up(); + static void print_buffer(std::ostream& ostream); + private: Console(); ~Console() = default; void deactivate_progress_bar(std::size_t idx, const std::string_view& msg = ""); - void print_progress(std::size_t idx); - bool skip_progress_bars() const; std::mutex m_mutex; - std::unique_ptr p_progress_manager; + std::unique_ptr p_progress_bar_manager; std::string json_hier; unsigned int json_index; nlohmann::json json_log; + static std::vector m_buffer; + friend class ProgressProxy; }; - inline void ProgressProxy::set_postfix(const std::string& s) - { - p_bar->set_postfix(s); - Console::instance().print_progress(m_idx); - } - class MessageLogger { public: @@ -279,11 +182,21 @@ namespace mamba std::stringstream& stream(); + static void activate_buffer(); + static void deactivate_buffer(); + static void print_buffer(std::ostream& ostream); + private: std::string m_file; int m_line; spdlog::level::level_enum m_level; std::stringstream m_stream; + + static std::mutex m_mutex; + static bool use_buffer; + static std::vector> m_buffer; + + static void emit(const std::string& msg, const spdlog::level::level_enum& level); }; diff --git a/libmamba/include/mamba/core/progress_bar.hpp b/libmamba/include/mamba/core/progress_bar.hpp index 1a9703d22..91aaf8486 100644 --- a/libmamba/include/mamba/core/progress_bar.hpp +++ b/libmamba/include/mamba/core/progress_bar.hpp @@ -7,12 +7,291 @@ #ifndef MAMBA_CORE_PROGRESS_BAR_HPP #define MAMBA_CORE_PROGRESS_BAR_HPP -#include +#include "spdlog/fmt/fmt.h" +#include "spdlog/fmt/bundled/color.h" + +#include +#include #include +#include +#include +#include +#include +#include + + +namespace cursor +{ + class CursorMovementTriple + { + public: + CursorMovementTriple(const char* esc, int n, const char* mod) + : m_esc(esc) + , m_mod(mod) + , m_n(n) + { + } + + const char* m_esc; + const char* m_mod; + int m_n; + }; + + inline std::ostream& operator<<(std::ostream& o, const CursorMovementTriple& m) + { + o << m.m_esc << m.m_n << m.m_mod; + return o; + } + + class CursorMod + { + public: + CursorMod(const char* mod) + : m_mod(mod) + { + } + + std::ostream& operator<<(std::ostream& o) + { + o << m_mod; + return o; + } + + const char* m_mod; + }; + + inline std::ostream& operator<<(std::ostream& o, const CursorMod& m) + { + o << m.m_mod; + return o; + } + + inline auto up(int n) + { + return CursorMovementTriple("\x1b[", n, "A"); + } + + inline auto down(int n) + { + return CursorMovementTriple("\x1b[", n, "B"); + } + + inline auto forward(int n) + { + return CursorMovementTriple("\x1b[", n, "C"); + } + + inline auto back(int n) + { + return CursorMovementTriple("\x1b[", n, "D"); + } + + inline auto next_line(int n) + { + return CursorMovementTriple("\x1b[", n, "E"); + } + + inline auto prev_line(int n) + { + return CursorMovementTriple("\x1b[", n, "F"); + } + + inline auto horizontal_abs(int n) + { + return CursorMovementTriple("\x1b[", n, "G"); + } + + inline auto home() + { + return CursorMod("\x1b[H"); + } + + inline auto erase_display(int n = 0) + { + return CursorMovementTriple("\x1b[", n, "J"); + } + + inline auto erase_line(int n = 0) + { + return CursorMovementTriple("\x1b[", n, "K"); + } + + inline auto scroll_up(int n = 1) + { + return CursorMovementTriple("\x1b[", n, "S"); + } + + inline auto scroll_down(int n = 1) + { + return CursorMovementTriple("\x1b[", n, "T"); + } + + inline auto show() + { + return CursorMod("\x1b[?25h"); + } + + inline auto hide() + { + return CursorMod("\x1b[?25l"); + } + + inline auto pos() + { + return CursorMod("\x1b[R"); + } + + inline auto delete_line(int n = 1) + { + return CursorMovementTriple("\x1b[", n, "M"); + } + + inline auto alternate_screen() + { + return CursorMod("\x1b[?1049h"); + } + + inline auto main_screen() + { + return CursorMod("\x1b[?1049l"); + } +} // namespace cursor namespace mamba { + void to_human_readable_filesize(std::ostream& o, double bytes, std::size_t precision = 0); + std::string to_human_readable_filesize(double bytes, std::size_t precision); + + std::ostream& write_duration(std::ostream& os, std::chrono::nanoseconds ns); + std::stringstream duration_stream(std::chrono::nanoseconds ns); + std::string duration_str(std::chrono::nanoseconds ns); + + int get_console_width(); + int get_console_height(); + + enum ChronoState + { + unset = 0, + started = 1, + paused = 2, + stopped = 3, + terminated = 4 + }; + + class Chrono + { + public: + using duration_t = std::chrono::milliseconds; + using time_point_t + = std::chrono::time_point; + + Chrono() = default; + + ChronoState status() const; + bool started() const; + bool paused() const; + bool stopped() const; + bool terminated() const; + bool unset() const; + + void start(); + void start(const time_point_t& time_point); + void pause(); + void resume(); + void stop(); + void terminate(); + + void set_start_time(const time_point_t& time_point); + time_point_t start_time() const; + time_point_t last_active_time(); + + void set_elapsed_time(const duration_t& time); + std::string elapsed_time_to_str(); + duration_t elapsed(); + + std::unique_lock chrono_lock(); + + private: + time_point_t m_start; + duration_t m_elapsed_ns = duration_t::zero(); + ChronoState m_state = ChronoState::unset; + std::mutex m_mutex; + + void compute_elapsed(); + + protected: + static time_point_t now(); + }; + + class FieldRepr + { + public: + bool active() const; + bool defined() const; + operator bool() const; + bool overflow() const; + + std::string formatted_value(bool allow_overflow = false) const; + std::size_t width(bool allow_overflow = true) const; + std::size_t stored_width() const; + std::string value() const; + + FieldRepr& activate(); + FieldRepr& deactivate(); + FieldRepr& set_format(const std::string& str); + FieldRepr& set_format(const std::string& str, std::size_t size); + FieldRepr& set_value(const std::string& str); + FieldRepr& set_width(std::size_t size); + FieldRepr& reset_width(); + FieldRepr& resize(std::size_t size); + + private: + std::string m_value = ""; + std::size_t m_width = 0; + std::string m_format = ""; + bool m_active = true; + + static std::string resize(const std::string& str, std::size_t size); + }; + class ProgressBar; + class ProgressBarManager; + + class ProgressBarRepr + { + public: + ProgressBarRepr() = default; + ProgressBarRepr(ProgressBar* pbar); + + FieldRepr prefix, progress, current, separator, total, speed, postfix, elapsed; + fmt::text_style style; + + void print(std::ostream& ostream, std::size_t width = 0, bool with_endl = true); + + void compute_progress(); + void compute_progress_width(); + void compute_progress_value(); + + ProgressBarRepr& set_width(std::size_t width); + std::size_t width() const; + + ProgressBarRepr& reset_fields(); + + const ProgressBar& progress_bar() const; + + private: + ProgressBar* p_progress_bar; + std::size_t m_width = 0; + + void set_same_widths(const ProgressBarRepr& r); + void deactivate_empty_fields(); + std::vector fields(); + + friend class ProgressBar; + friend class ProgressBarManager; + }; + + class ProgressBarManager; /******************************* * Public API of progress bars * @@ -22,7 +301,7 @@ namespace mamba { public: ProgressProxy() = default; - ProgressProxy(ProgressBar* ptr, std::size_t idx); + ProgressProxy(ProgressBar* ptr); ~ProgressProxy() = default; ProgressProxy(const ProgressProxy&) = default; @@ -30,22 +309,65 @@ namespace mamba ProgressProxy(ProgressProxy&&) = default; ProgressProxy& operator=(ProgressProxy&&) = default; - void set_full(); - void set_progress(size_t current, size_t total); - void elapsed_time_to_stream(std::stringstream& s); - void set_postfix(const std::string& s); - void mark_as_completed(const std::string_view& final_message = ""); - void mark_as_extracted(); + bool defined() const; + operator bool() const; + ProgressProxy& set_bar(ProgressBar* ptr); + + ProgressProxy& set_progress(std::size_t current, std::size_t total); + ProgressProxy& update_progress(std::size_t current, std::size_t total); + ProgressProxy& set_progress(double progress); + ProgressProxy& set_current(std::size_t current); + ProgressProxy& set_in_progress(std::size_t in_progress); + ProgressProxy& update_current(std::size_t current); + ProgressProxy& set_total(std::size_t total); + ProgressProxy& set_speed(std::size_t speed); + ProgressProxy& set_full(); + ProgressProxy& activate_spinner(); + ProgressProxy& deactivate_spinner(); + + std::size_t current() const; + std::size_t in_progress() const; + std::size_t total() const; + std::size_t speed() const; + std::size_t avg_speed(const std::chrono::milliseconds& ref_duration + = std::chrono::milliseconds::max()); + double progress() const; + bool completed() const; + + ProgressProxy& set_prefix(const std::string& text); + ProgressProxy& set_postfix(const std::string& text); + ProgressProxy& set_repr_hook(std::function f); + ProgressProxy& set_progress_hook(std::function f); + ProgressProxy& mark_as_completed(const std::chrono::milliseconds& delay + = std::chrono::milliseconds::zero()); + + std::string elapsed_time_to_str() const; + std::string prefix() const; + + ProgressBarRepr& update_repr(bool compute_progress = true); + const ProgressBarRepr& repr() const; + ProgressBarRepr& repr(); + ProgressProxy& print(std::ostream& stream, std::size_t width = 0, bool with_endl = true); + + ProgressProxy& start(); + ProgressProxy& pause(); + ProgressProxy& resume(); + ProgressProxy& stop(); + + bool started() const; + + int width() const; private: ProgressBar* p_bar; - std::size_t m_idx; + + friend class ProgressBarManager; }; - class ProgressBarManager + class ProgressBarManager : public Chrono { public: - virtual ~ProgressBarManager() = default; + virtual ~ProgressBarManager(); ProgressBarManager(const ProgressBarManager&) = delete; ProgressBarManager& operator=(const ProgressBarManager&) = delete; @@ -54,13 +376,53 @@ namespace mamba virtual ProgressProxy add_progress_bar(const std::string& name, size_t expected_total = 0) = 0; - virtual void print_progress(std::size_t idx) = 0; - virtual void deactivate_progress_bar(std::size_t idx, const std::string_view& msg = "") = 0; + virtual void clear_progress_bars(); + virtual void add_label(const std::string& label, const ProgressProxy& progress_bar); - virtual void print(const std::string_view& str, bool skip_progress_bars) = 0; + void watch_print(const duration_t& period = std::chrono::milliseconds(100)); + virtual std::size_t print(std::ostream& os, + std::size_t width = 0, + std::size_t max_lines = std::numeric_limits::max(), + bool with_endl = true) + = 0; + void start(); + void terminate(); + + void register_print_hook(std::function f); + void register_pre_start_hook(std::function f); + void register_post_stop_hook(std::function f); + + void compute_bars_progress(std::vector& bars); + + void activate_sorting(); + void deactivate_sorting(); protected: + using progress_bar_ptr = std::unique_ptr; + ProgressBarManager() = default; + ProgressBarManager(std::size_t width); + + duration_t m_period = std::chrono::milliseconds(100); + std::vector m_progress_bars = {}; + std::map> m_labels; + bool m_marked_to_terminate = false, m_watch_print_started = false; + bool m_sort_bars = false; + std::size_t m_width = 0; + + std::mutex m_mutex; + + ProgressBar* raw_bar(const ProgressProxy& progress_bar); + + void run(); + + std::vector> m_print_hooks; + std::vector> m_pre_start_hooks; + std::vector> m_post_stop_hooks; + + void erase_lines(std::ostream& ostream, std::size_t count); + void call_print_hooks(std::ostream& ostream); + void sort_bars(bool max_height_exceeded); }; enum ProgressBarMode @@ -71,110 +433,158 @@ namespace mamba std::unique_ptr make_progress_bar_manager(ProgressBarMode); - /****************************************** - * Internal use of progress bars, * - * should not be used directly by clients * - ******************************************/ + + /********************************* + * Internal use of progress bars * + *********************************/ class MultiBarManager : public ProgressBarManager { public: MultiBarManager(); + MultiBarManager(std::size_t width); virtual ~MultiBarManager() = default; ProgressProxy add_progress_bar(const std::string& name, size_t expected_total) override; - void print_progress(std::size_t idx) override; - void deactivate_progress_bar(std::size_t idx, const std::string_view& msg = "") override; - void print(const std::string_view& str, bool skip_progress_bars) override; - - private: - void print_progress(); - - using progress_bar_ptr = std::unique_ptr; - std::vector m_progress_bars; - std::vector m_active_progress_bars; - bool m_progress_started; + std::size_t print(std::ostream& os, + std::size_t width = 0, + std::size_t max_lines = std::numeric_limits::max(), + bool with_endl = true) override; }; class AggregatedBarManager : public ProgressBarManager { public: AggregatedBarManager(); + AggregatedBarManager(std::size_t width); virtual ~AggregatedBarManager() = default; - ProgressProxy add_progress_bar(const std::string& name, size_t expected_total) override; - void print_progress(std::size_t idx) override; - void deactivate_progress_bar(std::size_t idx, const std::string_view& msg = "") override; - - void print(const std::string_view& str, bool skip_progress_bars) override; + ProgressProxy add_progress_bar(const std::string& name, + std::size_t expected_total) override; void update_download_bar(std::size_t current_diff); void update_extract_bar(); + void add_label(const std::string& label, const ProgressProxy& progress_bar) override; + void clear_progress_bars() override; + + void activate_sub_bars(); + void deactivate_sub_bars(); + + ProgressBar* aggregated_bar(const std::string& label); + + std::size_t print(std::ostream& os, + std::size_t width = 0, + std::size_t max_lines = std::numeric_limits::max(), + bool with_endl = true) override; + private: - void print_progress(); + std::map m_aggregated_bars; + bool m_print_sub_bars = false; + bool is_complete() const; - std::chrono::time_point m_start_time; - - using progress_bar_ptr = std::unique_ptr; - std::vector m_progress_bars; - progress_bar_ptr p_download_bar; - progress_bar_ptr p_extract_bar; - size_t m_completed; - size_t m_extracted; - std::mutex m_main_mutex; - size_t m_current; - size_t m_total; - bool m_progress_started; + void update_aggregates_progress(); }; - class ProgressBar + class ProgressBar : public Chrono { public: - virtual ~ProgressBar() = default; + virtual ~ProgressBar(); ProgressBar(const ProgressBar&) = delete; ProgressBar& operator=(const ProgressBar&) = delete; ProgressBar(ProgressBar&&) = delete; ProgressBar& operator=(ProgressBar&&) = delete; - virtual void print() = 0; - virtual void set_full() = 0; - virtual void set_progress(size_t current, size_t total) = 0; - virtual void set_extracted() = 0; - void set_start(); - void set_postfix(const std::string& postfix_text); - void elapsed_time_to_stream(std::stringstream& s); - const std::string& prefix() const; + virtual void print(std::ostream& stream, std::size_t width = 0, bool with_endl = true) = 0; + + ProgressBarRepr& update_repr(bool compute_bar = true); + const ProgressBarRepr& repr() const; + ProgressBarRepr& repr(); + + ProgressBar& set_progress(std::size_t current, std::size_t total); + ProgressBar& update_progress(std::size_t current, std::size_t total); + ProgressBar& set_progress(double progress); + ProgressBar& set_current(std::size_t current); + ProgressBar& set_in_progress(std::size_t in_progress); + ProgressBar& update_current(std::size_t current); + ProgressBar& set_total(std::size_t total); + ProgressBar& set_speed(std::size_t speed); + ProgressBar& set_full(); + ProgressBar& activate_spinner(); + ProgressBar& deactivate_spinner(); + + ProgressBar& mark_as_completed(const std::chrono::milliseconds& delay + = std::chrono::milliseconds::zero()); + + std::size_t current() const; + std::size_t in_progress() const; + std::size_t total() const; + std::size_t speed() const; + std::size_t avg_speed(const std::chrono::milliseconds& ref_duration + = std::chrono::milliseconds::max()); + double progress() const; + bool completed() const; + bool is_spinner() const; + + const std::set& active_tasks() const; + std::set& active_tasks(); + const std::set& all_tasks() const; + std::set& all_tasks(); + ProgressBar& add_active_task(const std::string& name); + ProgressBar& add_task(const std::string& name); + std::string last_active_task(); + + ProgressBar& set_repr_hook(std::function f); + ProgressBar& set_progress_hook(std::function f); + + ProgressBar& set_prefix(const std::string& str); + ProgressBar& set_postfix(const std::string& str); + + std::string prefix() const; + + int width() const; protected: - ProgressBar(const std::string& prefix); + ProgressBar(const std::string& prefix, std::size_t total, int width = 0); - std::chrono::nanoseconds m_elapsed_ns; - std::chrono::time_point m_start_time; + double m_progress = 0.; + std::size_t m_current = 0; + std::size_t m_in_progress = 0; + std::size_t m_total = 0; + std::size_t m_speed = 0, m_avg_speed = 0, m_current_avg = 0; + int m_width = 0; - std::string m_prefix; - std::string m_postfix; - bool m_start_time_saved; - bool m_activate_bob; + std::set m_active_tasks = {}; + std::set m_all_tasks = {}; + std::string m_last_active_task = ""; + time_point_t m_task_time, m_avg_speed_time; + + ProgressBarRepr m_repr; + + bool m_is_spinner; + bool m_completed = false; + + std::mutex m_mutex; + + void run(); + + std::function p_repr_hook; + std::function p_progress_hook; + + ProgressBar& call_progress_hook(); + ProgressBar& call_repr_hook(); }; class DefaultProgressBar : public ProgressBar { public: - DefaultProgressBar(const std::string& prefix, int width_cap = 20); + DefaultProgressBar(const std::string& prefix, std::size_t total, int width = 0); virtual ~DefaultProgressBar() = default; - void print() override; - void set_full() override; - void set_progress(size_t current, size_t total) override; - void set_extracted() override; - - private: - size_t m_progress; - int m_width_cap; + void print(std::ostream& stream, std::size_t width = 0, bool with_endl = true) override; }; class HiddenProgressBar : public ProgressBar @@ -182,19 +592,12 @@ namespace mamba public: HiddenProgressBar(const std::string& prefix, AggregatedBarManager* manager, - size_t expected_total); + std::size_t total, + int width = 0); virtual ~HiddenProgressBar() = default; - void print() override; - void set_full() override; - void set_progress(size_t current, size_t total) override; - void set_extracted() override; - - private: - AggregatedBarManager* p_manager; - std::size_t m_current; - std::size_t m_total; + void print(std::ostream& stream, std::size_t width = 0, bool with_endl = true) override; }; -} +} // namespace mamba #endif diff --git a/libmamba/include/mamba/core/transaction.hpp b/libmamba/include/mamba/core/transaction.hpp index 249f59695..e90cdbed8 100644 --- a/libmamba/include/mamba/core/transaction.hpp +++ b/libmamba/include/mamba/core/transaction.hpp @@ -24,6 +24,7 @@ #include "package_cache.hpp" #include "package_handling.hpp" #include "prefix_data.hpp" +#include "progress_bar.hpp" #include "repo.hpp" #include "thread_utils.hpp" #include "transaction_context.hpp" @@ -56,6 +57,7 @@ namespace mamba bool extract_from_cache(); bool validate_extract(); const std::string& name() const; + std::size_t expected_size() const; auto validation_result() const; void clear_cache() const; @@ -80,7 +82,8 @@ namespace mamba std::string m_sha256, m_md5; std::size_t m_expected_size; - ProgressProxy m_progress_proxy; + bool m_has_progress_bars = false; + ProgressProxy m_download_bar, m_extract_bar; std::unique_ptr m_target; std::string m_url, m_name, m_channel, m_filename; @@ -89,6 +92,9 @@ namespace mamba std::future m_extract_future; VALIDATION_RESULT m_validation_result = VALIDATION_RESULT::UNDEFINED; + + std::function extract_repr(); + std::function extract_progress_callback(); }; class DownloadExtractSemaphore @@ -115,8 +121,9 @@ namespace mamba MTransaction(MPool& pool, const std::vector& specs_to_remove, const std::vector& specs_to_install, - MultiPackageCache& caches); - MTransaction(MSolver& solver, MultiPackageCache& caches); + MultiPackageCache& caches, + std::vector repos); + MTransaction(MSolver& solver, MultiPackageCache& caches, std::vector repos); ~MTransaction(); @@ -133,9 +140,9 @@ namespace mamba void init(); to_conda_type to_conda(); void log_json(); - bool fetch_extract_packages(std::vector& repos); + bool fetch_extract_packages(); bool empty(); - bool prompt(std::vector& repos); + bool prompt(); void print(); bool execute(PrefixData& prefix); bool filter(Solvable* s); @@ -150,6 +157,8 @@ namespace mamba MultiPackageCache m_multi_cache; const fs::path m_cache_path; std::vector m_to_install, m_to_remove; + std::vector m_repos; + History::UserRequest m_history_entry; Transaction* m_transaction; @@ -160,7 +169,8 @@ namespace mamba MTransaction create_explicit_transaction_from_urls(MPool& pool, const std::vector& urls, - MultiPackageCache& package_caches); + MultiPackageCache& package_caches, + std::vector& repos); } // namespace mamba #endif // MAMBA_TRANSACTION_HPP diff --git a/libmamba/include/mamba/core/util.hpp b/libmamba/include/mamba/core/util.hpp index 29d4409b0..1f660ba78 100644 --- a/libmamba/include/mamba/core/util.hpp +++ b/libmamba/include/mamba/core/util.hpp @@ -59,7 +59,6 @@ namespace mamba bool is_package_file(const std::string_view& fn); - void to_human_readable_filesize(std::ostream& o, double bytes, std::size_t precision = 0); bool lexists(const fs::path& p); bool lexists(const fs::path& p, std::error_code& ec); std::vector filter_dir(const fs::path& dir, const std::string& suffix); diff --git a/libmamba/src/api/channel_loader.cpp b/libmamba/src/api/channel_loader.cpp index 4382a1410..2433b35c2 100644 --- a/libmamba/src/api/channel_loader.cpp +++ b/libmamba/src/api/channel_loader.cpp @@ -46,6 +46,8 @@ namespace mamba int max_prio = static_cast(channel_urls.size()); std::string prev_channel_name; + Console::instance().init_progress_bar_manager(ProgressBarMode::multi); + if (ctx.experimental) { load_tokens(); @@ -86,7 +88,7 @@ namespace mamba // TODO load local channels even when offline if (!ctx.offline) { - multi_dl.download(true); + multi_dl.download(MAMBA_DOWNLOAD_FAILFAST); } std::vector repos; diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index 70e2a1f61..dd3103a79 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -348,7 +348,6 @@ namespace mamba LOG_WARNING << "No 'channels' specified"; } - std::vector repos; MPool pool; if (ctx.offline) @@ -431,23 +430,20 @@ namespace mamba throw std::runtime_error("UnsatisfiableError"); } - MTransaction trans(solver, package_caches); + std::vector repo_ptrs; + for (auto& r : repos) + repo_ptrs.push_back(&r); + + MTransaction trans(solver, package_caches, repo_ptrs); if (ctx.json) { trans.log_json(); } - // TODO this is not so great - std::vector repo_ptrs; - for (auto& r : repos) - { - repo_ptrs.push_back(&r); - } Console::stream(); - bool yes = trans.prompt(repo_ptrs); - if (yes) + if (trans.prompt()) { if (create_env && !Context::instance().dry_run) detail::create_target_directory(ctx.target_prefix); @@ -470,19 +466,18 @@ namespace mamba PrefixData prefix_data(ctx.target_prefix); fs::path pkgs_dirs(Context::instance().root_prefix / "pkgs"); MultiPackageCache pkg_caches({ pkgs_dirs }); - auto transaction = create_explicit_transaction_from_urls(pool, specs, pkg_caches); + + std::vector repos = {}; + auto transaction = create_explicit_transaction_from_urls(pool, specs, pkg_caches, repos); prefix_data.load(); prefix_data.add_virtual_packages(get_virtual_packages()); MRepo(pool, prefix_data); - std::vector repo_ptrs; - if (ctx.json) transaction.log_json(); - bool yes = transaction.prompt(repo_ptrs); - if (yes) + if (transaction.prompt()) { if (create_env && !Context::instance().dry_run) detail::create_target_directory(ctx.target_prefix); diff --git a/libmamba/src/api/remove.cpp b/libmamba/src/api/remove.cpp index f61f1c3a5..af9e82b1e 100644 --- a/libmamba/src/api/remove.cpp +++ b/libmamba/src/api/remove.cpp @@ -75,29 +75,25 @@ namespace mamba auto repo = MRepo(pool, prefix_data); repos.push_back(repo); - const fs::path pkgs_dirs(ctx.root_prefix / "pkgs"); - MultiPackageCache package_caches({ pkgs_dirs }); - - // TODO this is not so great std::vector repo_ptrs; for (auto& r : repos) - { repo_ptrs.push_back(&r); - } + + const fs::path pkgs_dirs(ctx.root_prefix / "pkgs"); + MultiPackageCache package_caches({ pkgs_dirs }); auto execute_transaction = [&](MTransaction& transaction) { if (ctx.json) transaction.log_json(); - bool yes = transaction.prompt(repo_ptrs); - if (yes) + if (transaction.prompt()) transaction.execute(prefix_data); }; if (force) { std::vector mspecs(specs.begin(), specs.end()); - auto transaction = MTransaction(pool, mspecs, {}, package_caches); + auto transaction = MTransaction(pool, mspecs, {}, package_caches, repo_ptrs); execute_transaction(transaction); } else @@ -124,7 +120,7 @@ namespace mamba solver.add_jobs(specs, solver_flag); solver.solve(); - MTransaction transaction(solver, package_caches); + MTransaction transaction(solver, package_caches, repo_ptrs); execute_transaction(transaction); } } diff --git a/libmamba/src/api/update.cpp b/libmamba/src/api/update.cpp index 5074e68d6..20383b56f 100644 --- a/libmamba/src/api/update.cpp +++ b/libmamba/src/api/update.cpp @@ -106,20 +106,19 @@ namespace mamba solver.solve(); - MTransaction transaction(solver, package_caches); - // TODO this is not so great std::vector repo_ptrs; for (auto& r : repos) { repo_ptrs.push_back(&r); } + MTransaction transaction(solver, package_caches, repo_ptrs); auto execute_transaction = [&](MTransaction& transaction) { if (ctx.json) transaction.log_json(); - bool yes = transaction.prompt(repo_ptrs); + bool yes = transaction.prompt(); if (yes) transaction.execute(prefix_data); }; diff --git a/libmamba/src/core/fetch.cpp b/libmamba/src/core/fetch.cpp index 6cfb8330e..e1461dad1 100644 --- a/libmamba/src/core/fetch.cpp +++ b/libmamba/src/core/fetch.cpp @@ -347,41 +347,62 @@ namespace mamba return nitems * size; } + std::function DownloadTarget::download_repr() + { + return [&](ProgressBarRepr& r) -> void { + r.current.set_value( + fmt::format("{:>7}", to_human_readable_filesize(m_progress_bar.current(), 1))); + + std::string total_str; + if (!m_progress_bar.total() + || (m_progress_bar.total() == std::numeric_limits::max())) + total_str = "??.?MB"; + else + total_str = to_human_readable_filesize(m_progress_bar.total(), 1); + r.total.set_value(fmt::format("{:>7}", total_str)); + + auto speed = m_progress_bar.speed(); + r.speed.set_value( + fmt::format("@ {:>7}/s", speed ? to_human_readable_filesize(speed, 1) : "??.?MB")); + + r.separator.set_value("/"); + }; + } + + std::chrono::steady_clock::time_point DownloadTarget::progress_throttle_time() const + { + return m_progress_throttle_time; + } + + void DownloadTarget::set_progress_throttle_time( + const std::chrono::steady_clock::time_point& time) + { + m_progress_throttle_time = time; + } + int DownloadTarget::progress_callback( void* f, curl_off_t total_to_download, curl_off_t now_downloaded, curl_off_t, curl_off_t) { auto* target = static_cast(f); - if (Context::instance().quiet || Context::instance().json) - { + auto now = std::chrono::steady_clock::now(); + if (now - target->progress_throttle_time() < std::chrono::milliseconds(50)) return 0; - } - - if ((total_to_download != 0 || target->m_expected_size != 0)) - { - std::stringstream postfix; - postfix << std::setw(6); - to_human_readable_filesize(postfix, now_downloaded); - postfix << " / "; - postfix << std::setw(6); - to_human_readable_filesize(postfix, total_to_download); - postfix << " ("; - postfix << std::setw(6); - to_human_readable_filesize(postfix, target->get_speed(), 2); - postfix << "/s)"; - target->m_progress_bar.set_progress(now_downloaded, total_to_download); - target->m_progress_bar.set_postfix(postfix.str()); - } else - { - std::stringstream postfix; - to_human_readable_filesize(postfix, now_downloaded); - postfix << " / ?? ("; - to_human_readable_filesize(postfix, target->get_speed(), 2); - postfix << "/s)"; - target->m_progress_bar.set_progress(SIZE_MAX, SIZE_MAX); - target->m_progress_bar.set_postfix(postfix.str()); - } + target->set_progress_throttle_time(now); + + if (!total_to_download && !target->expected_size()) + target->m_progress_bar.activate_spinner(); + else + target->m_progress_bar.deactivate_spinner(); + + if (!total_to_download && target->expected_size()) + target->m_progress_bar.update_current(now_downloaded); + else + target->m_progress_bar.update_progress(now_downloaded, total_to_download); + + target->m_progress_bar.set_speed(target->get_speed()); + return 0; } @@ -407,6 +428,8 @@ namespace mamba { m_has_progress_bar = true; m_progress_bar = progress_proxy; + m_progress_bar.set_repr_hook(download_repr()); + curl_easy_setopt(m_handle, CURLOPT_XFERINFOFUNCTION, &DownloadTarget::progress_callback); curl_easy_setopt(m_handle, CURLOPT_XFERINFODATA, this); curl_easy_setopt(m_handle, CURLOPT_NOPROGRESS, 0L); @@ -422,12 +445,16 @@ namespace mamba return m_name; } + std::size_t DownloadTarget::expected_size() const + { + return m_expected_size; + } + static size_t discard(char* ptr, size_t size, size_t nmemb, void*) { return size * nmemb; } - bool DownloadTarget::resource_exists() { auto handle = curl_easy_init(); @@ -474,7 +501,14 @@ namespace mamba { curl_off_t speed; CURLcode res = curl_easy_getinfo(m_handle, CURLINFO_SPEED_DOWNLOAD_T, &speed); - return res == CURLE_OK ? speed : 0; + if (res != CURLE_OK) + { + if (m_has_progress_bar) + speed = m_progress_bar.avg_speed(); + else + speed = 0; + } + return speed; } void DownloadTarget::set_result(CURLcode r) @@ -499,7 +533,8 @@ namespace mamba if (m_has_progress_bar) { - m_progress_bar.set_progress(0, 1); + m_progress_bar.update_progress(0, 1); + // m_progress_bar.set_elapsed_time(); m_progress_bar.set_postfix(curl_easy_strerror(result)); } if (!m_ignore_failure && !can_retry()) @@ -513,17 +548,12 @@ namespace mamba { char* effective_url = nullptr; - auto cres = curl_easy_getinfo(m_handle, CURLINFO_SPEED_DOWNLOAD_T, &avg_speed); - if (cres != CURLE_OK) - { - avg_speed = 0; - } - + avg_speed = get_speed(); curl_easy_getinfo(m_handle, CURLINFO_RESPONSE_CODE, &http_status); curl_easy_getinfo(m_handle, CURLINFO_EFFECTIVE_URL, &effective_url); curl_easy_getinfo(m_handle, CURLINFO_SIZE_DOWNLOAD_T, &downloaded_size); - LOG_INFO << "Transfer finalized, status: " << http_status << " [" << effective_url << "] " + LOG_INFO << "Transfer finalized, status " << http_status << " [" << effective_url << "] " << downloaded_size << " bytes"; if (http_status >= 500 && can_retry()) @@ -533,26 +563,53 @@ namespace mamba = std::chrono::steady_clock::now() + std::chrono::seconds(m_retry_wait_seconds); std::stringstream msg; msg << "Failed (" << http_status << "), retry in " << m_retry_wait_seconds << "s"; - m_progress_bar.set_progress(0, downloaded_size); - m_progress_bar.set_postfix(msg.str()); + if (m_has_progress_bar) + { + m_progress_bar.update_progress(0, downloaded_size); + m_progress_bar.set_postfix(msg.str()); + } return false; } m_file.close(); - final_url = effective_url; - if (m_finalize_callback) + + if (m_has_progress_bar) { - return m_finalize_callback(); + m_progress_bar.set_speed(avg_speed); + m_progress_bar.set_total(downloaded_size); + m_progress_bar.set_full(); + m_progress_bar.set_postfix("downloaded"); } + + bool ret = true; + if (m_finalize_callback) + ret = m_finalize_callback(); else { if (m_has_progress_bar) - { - m_progress_bar.mark_as_completed("Downloaded " + m_name); - } + m_progress_bar.mark_as_completed(); + else + Console::instance().print(name() + " completed"); } - return true; + + if (m_has_progress_bar) + { + // make sure total value is up-to-date + m_progress_bar.update_repr(false); + // select field to display and make sure they are + // properly set if not yet printed by the progress bar manager + ProgressBarRepr r = m_progress_bar.repr(); + r.prefix.set_format("{:<35}").reset_width(); + r.progress.deactivate(); + r.current.deactivate(); + r.separator.deactivate(); + + auto console_stream = Console::stream(); + r.print(console_stream, 0, false); + } + + return ret; } /************************************** @@ -622,7 +679,7 @@ namespace mamba if (msg->msg == CURLMSG_DONE) { - LOG_INFO << "Transfer done ..."; + LOG_INFO << "Transfer done for '" << current_target->name() << "'"; // We are only interested in messages about finished transfers curl_multi_remove_handle(m_handle, current_target->handle()); @@ -632,7 +689,7 @@ namespace mamba // transfer did not work! can we retry? if (current_target->can_retry()) { - LOG_INFO << "Adding target to retry!"; + LOG_INFO << "Setting retry for '" << current_target->name() << "'"; m_retry_targets.push_back(current_target); } else @@ -648,10 +705,40 @@ namespace mamba return true; } - bool MultiDownloadTarget::download(bool failfast) + bool MultiDownloadTarget::download(int options) { + bool failfast = options & MAMBA_DOWNLOAD_FAILFAST; + bool sort = options & MAMBA_DOWNLOAD_SORT; + + auto& ctx = Context::instance(); + + if (m_targets.empty()) + { + LOG_INFO << "All targets to download are cached"; + return true; + } + + if (sort) + std::sort(m_targets.begin(), + m_targets.end(), + [](DownloadTarget* a, DownloadTarget* b) -> bool { + return a->expected_size() > b->expected_size(); + }); + LOG_INFO << "Starting to download targets"; + auto& pbar_manager = Console::instance().progress_bar_manager(); + interruption_guard g([]() { Console::instance().progress_bar_manager().terminate(); }); + + // be sure the progress bar manager was not already started + // it would mean this code is part of a larger process using progress bars + bool pbar_manager_started = pbar_manager.started(); + if (!(ctx.no_progress_bars || ctx.json || ctx.quiet || pbar_manager_started)) + { + pbar_manager.start(); + pbar_manager.watch_print(); + } + int still_running, repeats = 0; const long max_wait_msecs = 1000; do @@ -723,6 +810,13 @@ namespace mamba curl_multi_cleanup(m_handle); return false; } + + if (!(ctx.no_progress_bars || ctx.json || ctx.quiet || pbar_manager_started)) + { + pbar_manager.terminate(); + pbar_manager.clear_progress_bars(); + } + return true; } } // namespace mamba diff --git a/libmamba/src/core/output.cpp b/libmamba/src/core/output.cpp index 2b2f23d9b..fce57c1e8 100644 --- a/libmamba/src/core/output.cpp +++ b/libmamba/src/core/output.cpp @@ -26,53 +26,6 @@ namespace mamba { - std::ostream& write_duration(std::ostream& os, std::chrono::nanoseconds ns) - { - using std::chrono::duration; - using std::chrono::duration_cast; - using std::chrono::hours; - using std::chrono::minutes; - using std::chrono::seconds; - - using days = duration>; - char fill = os.fill(); - os.fill('0'); - auto d = duration_cast(ns); - ns -= d; - auto h = duration_cast(ns); - ns -= h; - auto m = duration_cast(ns); - ns -= m; - auto s = duration_cast(ns); - if (d.count() > 0) - { - os << std::setw(2) << d.count() << "d:"; - } - if (h.count() > 0) - { - os << std::setw(2) << h.count() << "h:"; - } - os << std::setw(2) << m.count() << "m:" << std::setw(2) << s.count() << 's'; - os.fill(fill); - return os; - } - - int get_console_width() - { -#ifndef _WIN32 - struct winsize w; - ioctl(0, TIOCGWINSZ, &w); - return w.ws_col; -#else - - CONSOLE_SCREEN_BUFFER_INFO coninfo; - auto res = GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &coninfo); - return coninfo.dwSize.X; -#endif - - return -1; - } - std::string cut_repo_name(const std::string& full_url) { std::string remaining_url, scheme, auth, token; @@ -297,8 +250,8 @@ namespace mamba Console::Console() : m_mutex() - , p_progress_manager(make_progress_bar_manager(ProgressBarMode::multi)) { + init_progress_bar_manager(ProgressBarMode::multi); #ifdef _WIN32 // initialize ANSI codes on Win terminals auto hStdout = GetStdHandle(STD_OUTPUT_HANDLE); @@ -339,10 +292,10 @@ namespace mamba if (!(Context::instance().quiet || Context::instance().json) || force_print) { const std::lock_guard lock(instance().m_mutex); - if (instance().p_progress_manager) + + if (instance().p_progress_bar_manager && instance().p_progress_bar_manager->started()) { - instance().p_progress_manager->print(instance().hide_secrets(str), - instance().skip_progress_bars()); + instance().m_buffer.push_back(instance().hide_secrets(str)); } else { @@ -351,6 +304,17 @@ namespace mamba } } + std::vector Console::m_buffer({}); + + void Console::print_buffer(std::ostream& ostream) + { + for (auto& message : m_buffer) + ostream << message << "\n"; + + const std::lock_guard lock(instance().m_mutex); + m_buffer.clear(); + } + bool Console::prompt(const std::string_view& message, char fallback, std::istream& input_stream) { if (Context::instance().always_yes) @@ -399,35 +363,31 @@ namespace mamba ProgressProxy Console::add_progress_bar(const std::string& name, size_t expected_total) { - return p_progress_manager->add_progress_bar(name, expected_total); + if (Context::instance().no_progress_bars) + return ProgressProxy(); + else + return p_progress_bar_manager->add_progress_bar(name, expected_total); } - void Console::init_multi_progress(ProgressBarMode mode) + void Console::clear_progress_bars() { - p_progress_manager = make_progress_bar_manager(mode); + return p_progress_bar_manager->clear_progress_bars(); } - void Console::deactivate_progress_bar(std::size_t idx, const std::string_view& msg) + ProgressBarManager& Console::init_progress_bar_manager(ProgressBarMode mode) { - std::lock_guard lock(instance().m_mutex); - p_progress_manager->deactivate_progress_bar(idx, msg); + p_progress_bar_manager = make_progress_bar_manager(mode); + p_progress_bar_manager->register_print_hook(Console::print_buffer); + p_progress_bar_manager->register_print_hook(MessageLogger::print_buffer); + p_progress_bar_manager->register_pre_start_hook(MessageLogger::activate_buffer); + p_progress_bar_manager->register_post_stop_hook(MessageLogger::deactivate_buffer); + + return *p_progress_bar_manager; } - void Console::print_progress(std::size_t idx) + ProgressBarManager& Console::progress_bar_manager() { - if (skip_progress_bars()) - { - return; - } - - std::lock_guard lock(instance().m_mutex); - p_progress_manager->print_progress(idx); - } - - bool Console::skip_progress_bars() const - { - return Context::instance().quiet || Context::instance().json - || Context::instance().no_progress_bars; + return *p_progress_bar_manager; } std::string strip_file_prefix(const std::string& file) @@ -455,8 +415,25 @@ namespace mamba MessageLogger::~MessageLogger() { - auto str = Console::hide_secrets(m_stream.str()); - switch (m_level) + if (!use_buffer) + emit(m_stream.str(), m_level); + else + { + const std::lock_guard lock(m_mutex); + m_buffer.push_back({ m_stream.str(), m_level }); + } + } + + std::mutex MessageLogger::m_mutex; + + bool MessageLogger::use_buffer(false); + + std::vector> MessageLogger::m_buffer({}); + + void MessageLogger::emit(const std::string& msg, const spdlog::level::level_enum& level) + { + auto str = Console::hide_secrets(msg); + switch (level) { case spdlog::level::critical: SPDLOG_CRITICAL(prepend(str, "", std::string(4, ' ').c_str())); @@ -488,11 +465,31 @@ namespace mamba return m_stream; } + void MessageLogger::activate_buffer() + { + use_buffer = true; + } + + void MessageLogger::deactivate_buffer() + { + use_buffer = false; + } + + void MessageLogger::print_buffer(std::ostream& /*ostream*/) + { + for (auto& [msg, level] : m_buffer) + emit(msg, level); + + spdlog::apply_all([&](std::shared_ptr l) { l->flush(); }); + + const std::lock_guard lock(m_mutex); + m_buffer.clear(); + } + Logger::Logger(const std::string& pattern) : spdlog::logger(std::string("mamba"), std::make_shared()) { - // set_pattern("%^[%L %Y-%m-%d %T:%e]%$ %v"); set_pattern(pattern); } diff --git a/libmamba/src/core/progress_bar.cpp b/libmamba/src/core/progress_bar.cpp index fea4a62ea..5f4e9af01 100644 --- a/libmamba/src/core/progress_bar.cpp +++ b/libmamba/src/core/progress_bar.cpp @@ -1,337 +1,436 @@ +#include "mamba/core/progress_bar.hpp" + +#include "termcolor/termcolor.hpp" + +#include "spdlog/fmt/bundled/core.h" +#include "spdlog/fmt/bundled/format.h" +#include "spdlog/fmt/bundled/format-inl.h" +#include "spdlog/fmt/bundled/ostream.h" + #ifdef _WIN32 #include #else #include #endif +#include +#include +#include +#include #include - -#include "mamba/core/output.hpp" -#include "mamba/core/progress_bar.hpp" -#include "mamba/core/thread_utils.hpp" -#include "mamba/core/util.hpp" +#include +#include +#include +#include namespace mamba { - /***************** - * ProgressProxy * - *****************/ - - ProgressProxy::ProgressProxy(ProgressBar* ptr, std::size_t idx) - : p_bar(ptr) - , m_idx(idx) + void to_human_readable_filesize(std::ostream& o, double bytes, std::size_t precision) { - } - - void ProgressProxy::set_full() - { - p_bar->set_full(); - } - - void ProgressProxy::set_progress(size_t current, size_t total) - { - if (is_sig_interrupted()) + const char* sizes[] = { " B", "kB", "MB", "GB", "TB", "PB" }; + int order = 0; + while (bytes >= 1000 && order < (6 - 1)) { - return; + order++; + bytes = bytes / 1000; } - p_bar->set_progress(current, total); - Console::instance().print_progress(m_idx); + o << std::fixed << std::setprecision(precision) << bytes << sizes[order]; } - void ProgressProxy::elapsed_time_to_stream(std::stringstream& s) + std::string to_human_readable_filesize(double bytes, std::size_t precision) { - if (is_sig_interrupted()) - { - return; - } - p_bar->elapsed_time_to_stream(s); + std::stringstream o; + to_human_readable_filesize(o, bytes, precision); + return o.str(); } - void ProgressProxy::mark_as_completed(const std::string_view& final_message) + + namespace { - if (is_sig_interrupted()) + void print_formatted_field_repr(std::ostream& ostream, + FieldRepr& r, + std::size_t& current_width, + std::size_t max_width, + const std::string& sep = " ", + bool allow_overflow = false) { - return; + if (r && (!max_width || (current_width + r.width() <= max_width))) + { + ostream << sep << r.formatted_value(allow_overflow); + current_width += r.width(); + } + } + + void print_formatted_bar_repr(std::ostream& ostream, + ProgressBarRepr& r, + std::size_t width, + bool with_endl = true) + { + std::stringstream sstream; + std::size_t cumulated_width = 0; + + print_formatted_field_repr(sstream, r.prefix, cumulated_width, width, ""); + print_formatted_field_repr(sstream, r.progress, cumulated_width, width, " ", true); + + if (r.style.has_foreground()) + { + ostream << fmt::format(r.style, "{}", sstream.str()); + sstream.str(""); + } + + print_formatted_field_repr(sstream, r.current, cumulated_width, width); + print_formatted_field_repr(sstream, r.separator, cumulated_width, width); + print_formatted_field_repr(sstream, r.total, cumulated_width, width); + print_formatted_field_repr(sstream, r.speed, cumulated_width, width); + print_formatted_field_repr(sstream, r.postfix, cumulated_width, width); + print_formatted_field_repr(sstream, r.elapsed, cumulated_width, width); + + if (with_endl) + sstream << "\n"; + + if (r.style.has_foreground()) + ostream << fmt::format(r.style, "{}", sstream.str()); + else + ostream << fmt::format("{}", sstream.str()); + } + + void print_formatted_bar_repr(ProgressBarRepr& r, + std::size_t width, + bool with_endl = true, + bool flush = true) + { + print_formatted_bar_repr(std::cout, r, width, with_endl); + if (flush) + std::cout << std::flush; } - Console::instance().deactivate_progress_bar(m_idx, final_message); } - void ProgressProxy::mark_as_extracted() + /********** + * Chrono * + **********/ + + bool Chrono::started() const { - if (is_sig_interrupted()) - { - return; - } - p_bar->set_extracted(); - Console::instance().print_progress(m_idx); + return m_state == ChronoState::started; } - /********************** - * ProgressBarManager * - **********************/ - - std::unique_ptr make_progress_bar_manager(ProgressBarMode mode) + bool Chrono::paused() const { - if (mode == ProgressBarMode::multi) + return m_state == ChronoState::paused; + } + + bool Chrono::stopped() const + { + return m_state == ChronoState::stopped; + } + + bool Chrono::terminated() const + { + return m_state == ChronoState::terminated; + } + + bool Chrono::unset() const + { + return m_state == ChronoState::unset; + } + + ChronoState Chrono::status() const + { + return m_state; + } + + void Chrono::start() + { + start(now()); + } + + void Chrono::start(const time_point_t& time_point) + { + std::lock_guard lock(m_mutex); + m_start = time_point; + m_state = ChronoState::started; + } + + void Chrono::pause() + { + compute_elapsed(); + std::lock_guard lock(m_mutex); + m_state = ChronoState::paused; + } + + void Chrono::resume() + { + if (m_state != ChronoState::started) { - return std::make_unique(); + std::lock_guard lock(m_mutex); + m_state = ChronoState::started; + m_start = now() - m_elapsed_ns; } - return std::make_unique(); + } + + void Chrono::stop() + { + compute_elapsed(); + std::lock_guard lock(m_mutex); + m_state = ChronoState::stopped; + } + + void Chrono::terminate() + { + compute_elapsed(); + std::lock_guard lock(m_mutex); + m_state = ChronoState::terminated; + } + + auto Chrono::last_active_time() -> time_point_t + { + return m_start + m_elapsed_ns; + } + + auto Chrono::elapsed() -> duration_t + { + compute_elapsed(); + return m_elapsed_ns; + } + + void Chrono::set_elapsed_time(const duration_t& time) + { + std::lock_guard lock(m_mutex); + m_elapsed_ns = time; + m_start = now() - time; + } + + std::string Chrono::elapsed_time_to_str() + { + std::stringstream stream; + if (m_state != ChronoState::unset) + write_duration(stream, elapsed()); + else + stream << "--"; + + return stream.str(); + } + + auto Chrono::start_time() const -> time_point_t + { + return m_start; + } + + void Chrono::set_start_time(const time_point_t& time_point) + { + std::lock_guard lock(m_mutex); + m_elapsed_ns = m_start - time_point; + m_start = time_point; + } + + void Chrono::compute_elapsed() + { + if (m_state == ChronoState::started) + { + std::lock_guard lock(m_mutex); + m_elapsed_ns = now() - m_start; + } + } + + std::unique_lock Chrono::chrono_lock() + { + return std::unique_lock(m_mutex); + } + + auto Chrono::now() -> time_point_t + { + return std::chrono::time_point_cast(std::chrono::high_resolution_clock::now()); + } + + /************* + * FieldRepr * + *************/ + + bool FieldRepr::defined() const + { + return width() > 0; + } + + FieldRepr::operator bool() const + { + return defined(); + } + + bool FieldRepr::active() const + { + return m_active; + } + + FieldRepr& FieldRepr::activate() + { + m_active = true; + return *this; + } + + FieldRepr& FieldRepr::deactivate() + { + m_active = false; + return *this; + } + + bool FieldRepr::overflow() const + { + return m_value.size() > m_width; + } + + std::string FieldRepr::formatted_value(bool allow_overflow) const + { + auto w = width(); + std::string val; + + if (!allow_overflow && overflow()) + val = resize(m_value, w); + else + val = m_value; + + if (active() && w) + if (m_format.empty()) + return fmt::format("{:<{}}", val, w); + else + return fmt::format(m_format, val, w); + else + return ""; + } + + std::string FieldRepr::value() const + { + return m_value; + } + + std::size_t FieldRepr::width(bool allow_overflow) const + { + if (m_active) + { + if (m_width || !allow_overflow) + return m_width; + return m_value.size(); + } + else + return 0; + } + + std::size_t FieldRepr::stored_width() const + { + return m_width; + } + + FieldRepr& FieldRepr::set_format(const std::string& str) + { + m_format = str; + return *this; + } + + FieldRepr& FieldRepr::set_format(const std::string& str, std::size_t size) + { + m_format = str; + m_width = size; + return *this; + } + + FieldRepr& FieldRepr::set_value(const std::string& str) + { + m_value = str; + return *this; + } + + FieldRepr& FieldRepr::set_width(std::size_t size) + { + m_width = size; + return *this; + } + + FieldRepr& FieldRepr::reset_width() + { + m_width = 0; + return *this; + } + + FieldRepr& FieldRepr::resize(std::size_t size) + { + m_value = resize(m_value, size); + return *this; + } + + std::string FieldRepr::resize(const std::string& str, std::size_t size) + { + if (str.size() > size) + return str.substr(0, size - 2) + ".."; + else + return str; } /******************* - * MultiBarManager * + * ProgressBarRepr * *******************/ - MultiBarManager::MultiBarManager() - : m_progress_bars() - , m_active_progress_bars() - , m_progress_started(false) + ProgressBarRepr::ProgressBarRepr(ProgressBar* pbar) + : p_progress_bar(pbar) { } - ProgressProxy MultiBarManager::add_progress_bar(const std::string& name, size_t) + const ProgressBar& ProgressBarRepr::progress_bar() const { - std::string prefix = name; - prefix.resize(PREFIX_LENGTH - 1, ' '); - prefix += ' '; - - m_progress_bars.push_back(std::make_unique(prefix)); - - return ProgressProxy(m_progress_bars[m_progress_bars.size() - 1].get(), - m_progress_bars.size() - 1); + return *p_progress_bar; } - void MultiBarManager::print_progress(std::size_t idx) + ProgressBarRepr& ProgressBarRepr::set_width(std::size_t width) { - std::size_t cursor_up = m_active_progress_bars.size(); - if (m_progress_started && cursor_up > 0) - { - std::cout << cursor::up(cursor_up); - } - - auto it = std::find(m_active_progress_bars.begin(), - m_active_progress_bars.end(), - m_progress_bars[idx].get()); - if (it == m_active_progress_bars.end()) - { - m_active_progress_bars.push_back(m_progress_bars[idx].get()); - } - - print_progress(); - m_progress_started = true; + m_width = width; + return *this; } - void MultiBarManager::deactivate_progress_bar(std::size_t idx, const std::string_view& msg) + std::size_t ProgressBarRepr::width() const { - auto& ctx = Context::instance(); - if (ctx.no_progress_bars && !(ctx.json || ctx.quiet)) - { - std::cout << m_progress_bars[idx]->prefix() << " " << msg << '\n'; - } - - auto it = std::find(m_active_progress_bars.begin(), - m_active_progress_bars.end(), - m_progress_bars[idx].get()); - if (it == m_active_progress_bars.end()) - { - return; - } - - m_active_progress_bars.erase(it); - - if (ctx.no_progress_bars || ctx.json || ctx.quiet) - { - return; - } - - int ps = m_active_progress_bars.size(); - std::cout << cursor::up(ps + 1) << cursor::erase_line(); - if (msg.empty()) - { - m_progress_bars[idx]->print(); - std::cout << std::endl; - } - else - { - std::cout << msg << std::endl; - } - print_progress(); + return m_width; } - void MultiBarManager::print(const std::string_view& str, bool skip_progress_bars) + void ProgressBarRepr::print(std::ostream& ostream, std::size_t width, bool with_endl) { - if (m_progress_started && m_active_progress_bars.size()) - { - const auto& ps = m_active_progress_bars.size(); - std::cout << cursor::up(ps) << cursor::erase_line() << str << std::endl; - if (!skip_progress_bars) - { - print_progress(); - } - } - else - { - std::cout << str << std::endl; - } + print_formatted_bar_repr(ostream, *this, width, with_endl); } - void MultiBarManager::print_progress() + void ProgressBarRepr::set_same_widths(const ProgressBarRepr& r) { - for (auto& bar : m_active_progress_bars) - { - bar->print(); - std::cout << '\n'; - } - std::cout << std::flush; + prefix.set_width(r.prefix.width()); + progress.set_width(r.progress.width()); + current.set_width(r.current.width()); + separator.set_width(r.separator.width()); + total.set_width(r.total.width()); + speed.set_width(r.speed.width()); + postfix.set_width(r.postfix.width()); + elapsed.set_width(r.elapsed.width()); + + if (!r.current.active()) + current.deactivate(); + if (!r.separator.active()) + separator.deactivate(); + if (!r.total.active()) + total.deactivate(); + if (!r.speed.active()) + speed.deactivate(); + if (!r.postfix.active()) + postfix.deactivate(); + if (!r.elapsed.active()) + elapsed.deactivate(); } - /************************ - * AggregatedBarManager * - ************************/ - - AggregatedBarManager::AggregatedBarManager() - : m_progress_bars() - , p_download_bar(std::make_unique("Downloading ", 100)) - , p_extract_bar(std::make_unique("Extracting ", 100)) - , m_completed(0) - , m_extracted(0) - , m_main_mutex() - , m_current(0) - , m_total(0) - , m_progress_started(false) + void ProgressBarRepr::compute_progress() { + compute_progress_width(); + compute_progress_value(); } - ProgressProxy AggregatedBarManager::add_progress_bar(const std::string& name, - size_t expected_total) - { - if (!m_progress_started) - { - m_start_time = std::chrono::high_resolution_clock::now(); - } - std::string prefix = name; - prefix.resize(PREFIX_LENGTH - 1, ' '); - prefix += ' '; - - m_progress_bars.push_back( - std::make_unique(prefix, this, expected_total)); - m_total += expected_total; - - return ProgressProxy(m_progress_bars[m_progress_bars.size() - 1].get(), - m_progress_bars.size() - 1); - } - - void AggregatedBarManager::print_progress(std::size_t idx) - { - if (m_progress_started) - { - std::cout << cursor::up(2); - } - - if (!is_complete()) - { - m_progress_started = true; - } - print_progress(); - } - - void AggregatedBarManager::deactivate_progress_bar(std::size_t idx, const std::string_view& msg) - { - if (Context::instance().quiet || Context::instance().json) - { - return; - } - else if (Context::instance().no_progress_bars) - { - std::cout << msg << '\n'; - return; - } - - std::cout << cursor::up(2) << cursor::erase_line(); - std::cout << msg << std::endl; - ++m_completed; - if (m_completed == m_progress_bars.size()) - { - const std::lock_guard lock(m_main_mutex); - m_current = m_total; - p_download_bar->set_progress(m_total, m_total); - } - print_progress(); - } - - void AggregatedBarManager::print(const std::string_view& str, bool skip_progress_bars) - { - if (m_progress_started && !is_complete() && !skip_progress_bars) - { - std::cout << cursor::erase_line() << str << std::endl; - print_progress(); - } - else - { - std::cout << str << std::endl; - } - } - - void AggregatedBarManager::update_download_bar(std::size_t current_diff) - { - auto update_time = std::chrono::high_resolution_clock::now(); - const std::lock_guard lock(m_main_mutex); - m_current += current_diff; - auto diff_time - = std::chrono::duration_cast(update_time - m_start_time); - size_t speed = static_cast(m_current) / diff_time.count() * 1000; - p_download_bar->set_progress(m_current, m_total); - std::stringstream s; - s << std::setw(7); - to_human_readable_filesize(s, speed, 2); - s << "/s"; - p_download_bar->set_postfix(s.str()); - } - - void AggregatedBarManager::update_extract_bar() - { - const std::lock_guard lock(m_main_mutex); - size_t bars_number = m_progress_bars.size(); - int padding = std::to_string(bars_number).length(); - ++m_extracted; - p_extract_bar->set_progress(m_extracted, bars_number); - std::stringstream s; - s << std::setw(9 - padding) << m_extracted << " / " << bars_number; - p_extract_bar->set_postfix(s.str()); - } - - void AggregatedBarManager::print_progress() - { - const std::lock_guard lock(m_main_mutex); - if (m_progress_started) - { - p_download_bar->print(); - std::cout << '\n'; - p_extract_bar->print(); - std::cout << '\n'; - } - if (is_complete()) - { - m_progress_started = false; - } - } - - bool AggregatedBarManager::is_complete() const - { - return m_completed == m_progress_bars.size() && m_extracted == m_progress_bars.size(); - } - - /*************** - * ProgressBar * - ***************/ - namespace { class ProgressScaleWriter { public: - ProgressScaleWriter(int bar_width, + ProgressScaleWriter(std::size_t bar_width, const std::string& fill, const std::string& lead, const std::string& remainder) @@ -342,164 +441,1092 @@ namespace mamba { } - std::ostream& write(std::ostream& os, std::size_t progress) const + std::string repr(std::size_t progress, std::size_t in_progress = 0) const { - int pos = static_cast(progress * m_bar_width / 100.0); - for (int i = 0; i < m_bar_width; ++i) + auto current = static_cast(progress * m_bar_width / 100.0); + auto in_progress_pos + = static_cast((progress + in_progress) * m_bar_width / 100.0); + + std::ostringstream oss; + for (std::size_t i = 0; i < m_bar_width; ++i) { - if (i < pos) - { - os << m_fill; - } - else if (i == pos) - { - os << m_lead; - } + if ((i < current - 1 && current > 0) || (current == m_bar_width)) + oss << fmt::format( + fmt::fg(fmt::terminal_color::bright_magenta), "{}", m_fill); + else if (i == current - 1) + oss << fmt::format( + fmt::fg(fmt::terminal_color::bright_magenta), "{}", m_lead); + else if ((i < in_progress_pos - 1 && in_progress_pos > 0) + || in_progress_pos == m_bar_width) + oss << fmt::format(fmt::fg(fmt::terminal_color::bright_cyan), "{}", m_fill); + else if (i == in_progress_pos - 1 && i < m_bar_width - 1) + oss << fmt::format(fmt::fg(fmt::terminal_color::bright_cyan), "{}", m_lead); else - { - os << m_remainder; - } + oss << fmt::format( + fmt::fg(fmt::terminal_color::bright_black), "{}", m_remainder); } - return os; + return oss.str(); } private: - int m_bar_width; + std::size_t m_bar_width; std::string m_fill; std::string m_lead; std::string m_remainder; }; - } + } // namespace - ProgressBar::ProgressBar(const std::string& prefix) - : m_prefix(prefix) - , m_start_time_saved(false) - , m_activate_bob(false) + void ProgressBarRepr::compute_progress_width() { - } + std::size_t max_width; - void ProgressBar::set_start() - { - m_start_time = std::chrono::high_resolution_clock::now(); - m_start_time_saved = true; - } - - - void ProgressBar::set_postfix(const std::string& postfix_text) - { - m_postfix = postfix_text; - } - - const std::string& ProgressBar::prefix() const - { - return m_prefix; - } - - void ProgressBar::elapsed_time_to_stream(std::stringstream& s) - { - if (m_start_time_saved) + if (m_width) + max_width = m_width; + else { - auto now = std::chrono::high_resolution_clock::now(); - m_elapsed_ns = std::chrono::duration_cast(now - m_start_time); - s << "("; - write_duration(s, m_elapsed_ns); - s << ") "; + int console_width = get_console_width(); + if (console_width != -1) + max_width = console_width; + else + max_width = 100; + } + + progress.set_width(40); + std::size_t total_width = prefix.width() + progress.width() + current.width() + + separator.width() + total.width() + speed.width() + + postfix.width() + elapsed.width() + 1; + + // Add extra whitespaces between fields (prefix, progress, + // and elasped fields are assumed always displayed) + if (current) + total_width += 1; + if (separator) + total_width += 1; + if (total) + total_width += 1; + if (speed) + total_width += 1; + if (postfix) + total_width += 1; + if (elapsed) + total_width += 1; + + // Reduce some fields to fit console width + // 1: reduce bar width + if (max_width < total_width && progress) + { + total_width -= progress.width() - 15; + progress.set_width(15); + } + // 2: remove the total value and the separator + if (max_width < total_width && total) + { + total_width -= total.width() + separator.width() + 2; + total.deactivate(); + separator.deactivate(); + } + // 3: remove the speed + if (max_width < total_width && speed) + { + total_width -= speed.width() + 1; + speed.deactivate(); + } + // 4: remove the postfix + if (max_width < total_width && postfix) + { + total_width -= postfix.width() + 1; + postfix.deactivate(); + } + std::size_t prefix_min_width = prefix.width(); + // 5: truncate the prefix if too long + if (max_width < total_width && prefix.width() > 20 && prefix) + { + // keep a minimal size to make it readable + total_width -= prefix.width() - 20; + prefix.set_width(20); + } + // 6: display progress without a bar + if (max_width < total_width && progress) + { + // keep capability to display progress up to "100%" + total_width -= progress.width() - 4; + progress.set_width(4); + } + // 7: remove the current value + if (max_width < total_width && current) + { + total_width -= current.width() + 1; + current.deactivate(); + } + // 8: remove the elapsed time + if (max_width < total_width && elapsed) + { + total_width -= elapsed.width() + 1; + elapsed.deactivate(); + } + + // Redistribute available space + // 1: start with the prefix if it was shrinked + if (total_width < max_width && prefix && prefix.width() < prefix_min_width) + { + if ((max_width - total_width) < (prefix_min_width - prefix.width())) + { + prefix.set_width(prefix.width() + (max_width - total_width)); + total_width = max_width; + } + else + { + total_width += prefix_min_width - prefix.width(); + prefix.set_width(prefix_min_width); + } + } + // 2: give the remaining free space to the progress bar + if (total_width < max_width) + { + progress.set_width(progress.width() + (max_width - total_width)); + total_width = max_width; + } + } + + std::vector ProgressBarRepr::fields() + { + return { &prefix, &progress, ¤t, &separator, &total, &speed, &postfix, &elapsed }; + } + + ProgressBarRepr& ProgressBarRepr::reset_fields() + { + for (auto& f : fields()) + f->set_format("{:>{}}").activate().set_width(0); + prefix.set_format("{:<{}}"); + + return *this; + } + + void ProgressBarRepr::compute_progress_value() + { + std::stringstream sstream; + auto width = progress.width(false); + + if (!p_progress_bar->is_spinner()) + { + if (width < 12) + sstream << std::ceil(p_progress_bar->progress()) << "%"; + else + { + ProgressScaleWriter w{ width, "━", "╸", "━" }; + double in_progress = static_cast(p_progress_bar->in_progress()) + / static_cast(p_progress_bar->total()) * 100.; + sstream << w.repr(p_progress_bar->progress(), in_progress); + } } else { - s << "(--:--) "; + if (width < 12) + { + std::vector spinner + = { "⣾", "⣽", "⣻", "⢿", "⣿", "⡿", "⣟", "⣯", "⣷", "⣿" }; + auto pos + = (static_cast(std::round(p_progress_bar->progress())) % 50) / 5; + sstream << fmt::format("{:^4}", spinner[pos]); + } + else + { + auto pos = static_cast( + std::round(p_progress_bar->progress() * (width - 1) / 100.0)); + + std::size_t current_pos = 0, in_progress_pos = 0; + + if (p_progress_bar->total()) + { + current_pos = static_cast( + std::floor(static_cast(p_progress_bar->current()) + / static_cast(p_progress_bar->total()) * width)); + in_progress_pos = static_cast( + std::ceil(static_cast(p_progress_bar->current() + + p_progress_bar->in_progress()) + / static_cast(p_progress_bar->total()) * width)); + } + + auto spinner_half_width = width / 8; + for (std::size_t i = 0; i < width; ++i) + { + if ((i < current_pos - 1 && current_pos > 0) || (current_pos == width)) + { + sstream << fmt::format( + fmt::fg(fmt::terminal_color::bright_magenta), "{}", "━"); + } + else if (i == current_pos - 1) + { + sstream << fmt::format( + fmt::fg(fmt::terminal_color::bright_magenta), "{}", "╸"); + } + else if (in_progress_pos) + { + if ((i < in_progress_pos - 1) || (in_progress_pos == width)) + { + sstream << fmt::format( + fmt::fg(fmt::terminal_color::bright_cyan), "{}", "━"); + } + else if ((i == in_progress_pos - 1) && (i < width - 1)) + { + sstream << fmt::format( + fmt::fg(fmt::terminal_color::bright_cyan), "{}", "╸"); + } + else + { + sstream << fmt::format( + fmt::fg(fmt::terminal_color::bright_black), "{}", "━"); + } + } + else + { + if (i + spinner_half_width >= pos && i <= pos + spinner_half_width) + { + sstream << fmt::format( + fmt::fg(fmt::terminal_color::bright_magenta), "{}", "━"); + } + else if ((pos < spinner_half_width) + && (i >= width + pos - spinner_half_width)) + { + sstream << fmt::format( + fmt::fg(fmt::terminal_color::bright_magenta), "{}", "━"); + } + else if ((pos + spinner_half_width >= width) + && (i <= pos + spinner_half_width - width)) + { + sstream << fmt::format( + fmt::fg(fmt::terminal_color::bright_magenta), "{}", "━"); + } + else if ((i + spinner_half_width + 1 == pos) + || ((pos < spinner_half_width) + && (i == width + pos - spinner_half_width - 1))) + { + sstream << fmt::format( + fmt::fg(fmt::terminal_color::bright_magenta), "{}", "╺"); + } + else if ((i == pos + spinner_half_width + 1) + || ((pos + spinner_half_width + 1 > width) + && (i == pos + spinner_half_width + 1 - width))) + { + sstream << fmt::format( + fmt::fg(fmt::terminal_color::bright_magenta), "{}", "╸"); + } + else + { + sstream << fmt::format( + fmt::fg(fmt::terminal_color::bright_black), "{}", "━"); + } + } + } + } } + + progress.set_value(sstream.str()); + } + + /***************** + * ProgressProxy * + *****************/ + + ProgressProxy::ProgressProxy(ProgressBar* ptr) + : p_bar(ptr) + { + } + + bool ProgressProxy::defined() const + { + return p_bar != nullptr; + } + + ProgressProxy::operator bool() const + { + return p_bar != nullptr; + } + + ProgressProxy& ProgressProxy::set_bar(ProgressBar* ptr) + { + p_bar = ptr; + return *this; + } + + ProgressProxy& ProgressProxy::set_progress(std::size_t current, std::size_t total) + { + p_bar->set_progress(current, total); + return *this; + } + + ProgressProxy& ProgressProxy::update_progress(std::size_t current, std::size_t total) + { + p_bar->update_progress(current, total); + return *this; + } + + ProgressProxy& ProgressProxy::set_progress(double progress) + { + p_bar->set_progress(progress); + return *this; + } + + ProgressProxy& ProgressProxy::set_current(std::size_t current) + { + p_bar->set_progress(current); + return *this; + } + + ProgressProxy& ProgressProxy::set_in_progress(std::size_t in_progress) + { + p_bar->set_in_progress(in_progress); + return *this; + } + + ProgressProxy& ProgressProxy::update_current(std::size_t current) + { + p_bar->update_current(current); + return *this; + } + + ProgressProxy& ProgressProxy::set_total(std::size_t total) + { + p_bar->set_total(total); + return *this; + } + + ProgressProxy& ProgressProxy::set_full() + { + p_bar->set_full(); + return *this; + } + + ProgressProxy& ProgressProxy::set_speed(std::size_t speed) + { + p_bar->set_speed(speed); + return *this; + } + + ProgressProxy& ProgressProxy::activate_spinner() + { + p_bar->activate_spinner(); + return *this; + } + + ProgressProxy& ProgressProxy::deactivate_spinner() + { + p_bar->deactivate_spinner(); + return *this; + } + + std::size_t ProgressProxy::current() const + { + return p_bar->current(); + } + + std::size_t ProgressProxy::in_progress() const + { + return p_bar->in_progress(); + } + + std::size_t ProgressProxy::total() const + { + return p_bar->total(); + } + + std::size_t ProgressProxy::speed() const + { + return p_bar->speed(); + } + + std::size_t ProgressProxy::avg_speed(const std::chrono::milliseconds& duration) + { + return p_bar->avg_speed(duration); + } + + double ProgressProxy::progress() const + { + return p_bar->progress(); + } + + bool ProgressProxy::completed() const + { + return p_bar->completed(); + } + + ProgressProxy& ProgressProxy::start() + { + p_bar->start(); + return *this; + } + + ProgressProxy& ProgressProxy::stop() + { + p_bar->stop(); + return *this; + } + + ProgressProxy& ProgressProxy::pause() + { + p_bar->pause(); + return *this; + } + + ProgressProxy& ProgressProxy::resume() + { + p_bar->resume(); + return *this; + } + + bool ProgressProxy::started() const + { + return p_bar->started(); + } + + ProgressProxy& ProgressProxy::mark_as_completed(const std::chrono::milliseconds& delay) + { + p_bar->mark_as_completed(delay); + return *this; + } + + ProgressProxy& ProgressProxy::set_prefix(const std::string& str) + { + p_bar->set_prefix(str); + return *this; + } + + ProgressProxy& ProgressProxy::set_postfix(const std::string& text) + { + p_bar->set_postfix(text); + return *this; + } + + ProgressProxy& ProgressProxy::set_repr_hook(std::function f) + { + p_bar->set_repr_hook(f); + return *this; + } + + ProgressProxy& ProgressProxy::set_progress_hook(std::function f) + { + p_bar->set_progress_hook(f); + return *this; + } + + std::string ProgressProxy::elapsed_time_to_str() const + { + return p_bar->elapsed_time_to_str(); + } + + std::string ProgressProxy::prefix() const + { + return p_bar->prefix(); + } + + int ProgressProxy::width() const + { + return p_bar->width(); + } + + ProgressProxy& ProgressProxy::print(std::ostream& stream, std::size_t width, bool with_endl) + { + p_bar->print(stream, width, with_endl); + return *this; + } + + ProgressBarRepr& ProgressProxy::update_repr(bool compute_progress) + { + return p_bar->update_repr(compute_progress); + } + + const ProgressBarRepr& ProgressProxy::repr() const + { + return p_bar->repr(); + } + + ProgressBarRepr& ProgressProxy::repr() + { + return p_bar->repr(); + } + + /********************** + * ProgressBarManager * + **********************/ + + ProgressBarManager::ProgressBarManager(std::size_t width) + : m_width(width) + { + } + + ProgressBarManager::~ProgressBarManager() + { + if (m_watch_print_started) + terminate(); + } + + std::unique_ptr make_progress_bar_manager(ProgressBarMode mode) + { + if (mode == ProgressBarMode::multi) + { + return std::make_unique(); + } + return std::make_unique(); + } + + void ProgressBarManager::clear_progress_bars() + { + std::lock_guard lock(m_mutex); + m_labels.clear(); + m_progress_bars.clear(); + } + + ProgressBar* ProgressBarManager::raw_bar(const ProgressProxy& progress_bar) + { + return progress_bar.p_bar; + } + + void ProgressBarManager::erase_lines(std::ostream& ostream, std::size_t count) + { + for (std::size_t i = 0; i < count; ++i) + ostream << cursor::erase_line(2) << cursor::up(1); + + call_print_hooks(ostream); + } + + void ProgressBarManager::call_print_hooks(std::ostream& ostream) + { + ostream << cursor::erase_line(2) << cursor::horizontal_abs(0); + for (auto& f : m_print_hooks) + f(ostream); + } + + void ProgressBarManager::compute_bars_progress(std::vector& bars) + { + if (!bars.empty()) + { + std::size_t prefix_w = 0, current_w = 0, separator_w = 0, total_w = 0, speed_w = 0, + postfix_w = 0, elapsed_w = 0; + + for (auto& b : bars) + { + auto& r = b->repr(); + r.reset_fields().set_width(m_width); + b->update_repr(false); + + prefix_w = std::max(r.prefix.value().size(), prefix_w); + current_w = std::max(r.current.value().size(), current_w); + separator_w = std::max(r.separator.value().size(), separator_w); + total_w = std::max(r.total.value().size(), total_w); + speed_w = std::max(r.speed.value().size(), speed_w); + postfix_w = std::max(r.postfix.value().size(), postfix_w); + elapsed_w = std::max(r.elapsed.value().size(), elapsed_w); + } + + auto& r0 = bars[0]->repr(); + r0.prefix.set_width(prefix_w); + r0.current.set_width(current_w); + r0.separator.set_width(separator_w); + r0.total.set_width(total_w); + r0.speed.set_width(speed_w); + r0.postfix.set_width(postfix_w); + r0.elapsed.set_width(elapsed_w); + r0.compute_progress(); + + for (auto& b : bars) + { + b->repr().set_same_widths(r0); + b->repr().compute_progress_value(); + } + } + } + + void ProgressBarManager::run() + { + auto time = start_time(); + bool watch = m_period > duration_t::zero(); + std::size_t previously_printed = 0; + std::cout << cursor::hide(); + + do + { + std::stringstream ostream; + auto duration = time - start_time(); + + erase_lines(ostream, previously_printed); + if (m_marked_to_terminate) + { + std::cout << ostream.str() << cursor::show() << std::flush; + m_marked_to_terminate = false; + break; + } + + ostream << "[+] " << std::fixed << std::setprecision(1) << duration_str(duration) + << "\n"; + previously_printed + = std::max(print(ostream, 0, get_console_height() - 1, false), std::size_t(1)); + std::cout << ostream.str() << std::flush; + + auto now = std::chrono::high_resolution_clock::now(); + while (now > time) + time += m_period; + + if (watch) + std::this_thread::sleep_until(time); + } while (started() && watch); + + m_watch_print_started = false; + } + + void ProgressBarManager::watch_print(const duration_t& period) + { + m_period = period; + start(); + m_marked_to_terminate = false; + m_watch_print_started = true; + std::thread t([&]() { run(); }); + t.detach(); + } + + void ProgressBarManager::start() + { + for (auto& f : m_pre_start_hooks) + f(); + + Chrono::start(); + } + + void ProgressBarManager::terminate() + { + if (!terminated()) + { + if (m_watch_print_started) + { + m_marked_to_terminate = true; + while (m_marked_to_terminate) + std::this_thread::sleep_for(m_period / 2); + } + + Chrono::terminate(); + + for (auto& f : m_post_stop_hooks) + f(); + } + } + + void ProgressBarManager::add_label(const std::string& label, const ProgressProxy& progress_bar) + { + std::lock_guard lock(m_mutex); + for (auto& p : m_progress_bars) + if (p.get() == raw_bar(progress_bar)) + { + if (m_labels.count(label) == 0) + m_labels.insert({ label, { raw_bar(progress_bar) } }); + else + m_labels[label].push_back(raw_bar(progress_bar)); + + break; + } + } + + void ProgressBarManager::register_print_hook(std::function f) + { + m_print_hooks.push_back(f); + } + + void ProgressBarManager::register_pre_start_hook(std::function f) + { + m_pre_start_hooks.push_back(f); + } + + void ProgressBarManager::register_post_stop_hook(std::function f) + { + m_post_stop_hooks.push_back(f); + } + + void ProgressBarManager::activate_sorting() + { + m_sort_bars = true; + } + + void ProgressBarManager::deactivate_sorting() + { + m_sort_bars = false; + } + + void ProgressBarManager::sort_bars(bool max_height_exceeded) + { + if (!max_height_exceeded) + std::sort(m_progress_bars.begin(), m_progress_bars.end(), [](auto& a, auto& b) { + return a->prefix() > b->prefix(); + }); + else + std::sort(m_progress_bars.begin(), m_progress_bars.end(), [](auto& a, auto& b) { + if (!a->started() && b->started()) + return false; + if (!b->started() && a->started()) + return true; + if (a->status() == ChronoState::unset && b->status() != ChronoState::unset) + return true; + if (b->status() == ChronoState::unset && a->status() != ChronoState::unset) + return false; + return a->last_active_time() > b->last_active_time(); + }); + } + + /*************** + * ProgressBar * + ***************/ + + ProgressBar::ProgressBar(const std::string& prefix, std::size_t total, int width) + : m_progress(0) + , m_total(total) + , m_width(width) + , m_repr(this) + , m_is_spinner(false) + { + m_repr.prefix.set_value(prefix); + } + + ProgressBar::~ProgressBar() + { + terminate(); + std::lock_guard lock(m_mutex); + } + + ProgressBar& ProgressBar::set_progress(std::size_t current, std::size_t total) + { + m_current = current; + m_total = total; + + if (!m_is_spinner && total && total != std::numeric_limits::max()) + { + if (current < total) + m_progress = static_cast(current) / static_cast(total) * 100.; + else + set_full(); + } + else + { + m_progress = (static_cast(m_progress) + 5) % 100; + } + return *this; + } + + ProgressBar& ProgressBar::update_progress(std::size_t current, std::size_t total) + { + if (!started()) + start(); + + set_progress(current, total); + return *this; + } + + ProgressBar& ProgressBar::set_progress(double progress) + { + m_progress = progress; + m_current = m_total * progress / 100.; + set_current(m_current); + return *this; + } + + ProgressBar& ProgressBar::set_current(std::size_t current) + { + set_progress(current, m_total); + return *this; + } + + ProgressBar& ProgressBar::set_in_progress(std::size_t in_progress) + { + m_in_progress = in_progress; + return *this; + } + + ProgressBar& ProgressBar::update_current(std::size_t current) + { + update_progress(current, m_total); + return *this; + } + + ProgressBar& ProgressBar::set_total(std::size_t total) + { + set_progress(m_current, total); + return *this; + } + + ProgressBar& ProgressBar::set_full() + { + if (m_total && m_total < std::numeric_limits::max()) + m_current = m_total; + else + m_total = m_current; + m_is_spinner = false; + m_progress = 100.; + return *this; + } + + ProgressBar& ProgressBar::set_speed(std::size_t speed) + { + m_speed = speed; + return *this; + } + + ProgressBar& ProgressBar::activate_spinner() + { + if (!m_is_spinner) + { + unsigned seed = std::chrono::system_clock::now().time_since_epoch().count(); + std::default_random_engine generator(seed); + std::uniform_int_distribution distribution(0, 100); + m_progress = distribution(generator); + } + m_is_spinner = true; + return *this; + } + + ProgressBar& ProgressBar::deactivate_spinner() + { + if (m_current < m_total && m_total) + m_progress = static_cast(m_current) / static_cast(m_total) * 100.; + else + set_full(); + m_is_spinner = false; + return *this; + } + + std::size_t ProgressBar::current() const + { + return m_current; + } + + std::size_t ProgressBar::in_progress() const + { + return m_in_progress; + } + + std::size_t ProgressBar::total() const + { + return m_total; + } + + std::size_t ProgressBar::speed() const + { + return m_speed; + } + + std::size_t ProgressBar::avg_speed(const std::chrono::milliseconds& ref_duration) + { + if (started()) + { + auto now = Chrono::now(); + auto elapsed_since_last_avg + = std::chrono::duration_cast(now - m_avg_speed_time); + auto total_elapsed = std::chrono::duration_cast(elapsed()); + + if (ref_duration <= elapsed_since_last_avg && elapsed_since_last_avg.count()) + { + if (total_elapsed < ref_duration && total_elapsed.count()) + m_avg_speed = m_current / total_elapsed.count() * 1000; + else + m_avg_speed + = (m_current - m_current_avg) / elapsed_since_last_avg.count() * 1000; + m_avg_speed_time = now; + m_current_avg = m_current; + } + } + else + m_avg_speed = 0; + + return m_avg_speed; + } + + double ProgressBar::progress() const + { + return m_progress; + } + + bool ProgressBar::completed() const + { + return m_completed; + } + + bool ProgressBar::is_spinner() const + { + return m_is_spinner; + } + + const std::set& ProgressBar::active_tasks() const + { + return m_active_tasks; + } + + std::set& ProgressBar::active_tasks() + { + return m_active_tasks; + } + + const std::set& ProgressBar::all_tasks() const + { + return m_all_tasks; + } + + std::set& ProgressBar::all_tasks() + { + return m_all_tasks; + } + + std::string ProgressBar::last_active_task() + { + auto now = Chrono::now(); + if ((now - m_task_time < std::chrono::seconds(1)) && !m_last_active_task.empty() + && m_active_tasks.count(m_last_active_task)) + return m_last_active_task; + + m_task_time = now; + if (m_active_tasks.empty()) + m_last_active_task = ""; + else if (m_active_tasks.size() == 1) + m_last_active_task = *m_active_tasks.begin(); + else + { + auto it = m_active_tasks.find(m_last_active_task); + if (std::distance(it, m_active_tasks.end()) <= 1) + m_last_active_task = *m_active_tasks.begin(); + else + m_last_active_task = *it; + } + return m_last_active_task; + } + + ProgressBar& ProgressBar::add_active_task(const std::string& name) + { + m_active_tasks.insert(name); + m_all_tasks.insert(name); + return *this; + } + + ProgressBar& ProgressBar::add_task(const std::string& name) + { + m_all_tasks.insert(name); + return *this; + } + + ProgressBar& ProgressBar::mark_as_completed(const std::chrono::milliseconds& delay) + { + pause(); + set_full(); + + time_point_t stop_time_point = now() + delay; + + if (delay.count()) + { + std::thread t( + [&](const time_point_t& stop_time_point) { + std::lock_guard lock(m_mutex); + while (now() < stop_time_point && status() < ChronoState::stopped) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + this->m_completed = true; + stop(); + }, + stop_time_point); + t.detach(); + } + else + { + stop(); + m_completed = true; + } + return *this; + } + + ProgressBar& ProgressBar::set_prefix(const std::string& str) + { + m_repr.prefix.set_value(str); + return *this; + } + + ProgressBar& ProgressBar::set_postfix(const std::string& str) + { + m_repr.postfix.set_value(str); + return *this; + } + + ProgressBar& ProgressBar::set_repr_hook(std::function f) + { + p_repr_hook = f; + return *this; + } + + ProgressBar& ProgressBar::set_progress_hook(std::function f) + { + p_progress_hook = f; + return *this; + } + + ProgressBar& ProgressBar::call_progress_hook() + { + if (p_progress_hook != nullptr) + { + auto proxy = ProgressProxy(this); + p_progress_hook(proxy); + } + return *this; + } + + ProgressBar& ProgressBar::call_repr_hook() + { + if (p_repr_hook != nullptr) + p_repr_hook(m_repr); + return *this; + } + + std::string ProgressBar::prefix() const + { + return m_repr.prefix.value(); + } + + int ProgressBar::width() const + { + return m_width; + } + + ProgressBarRepr& ProgressBar::update_repr(bool compute_progress) + { + call_progress_hook(); + m_repr.elapsed.set_value(fmt::format("{:>5}", elapsed_time_to_str())); + call_repr_hook(); + + if (compute_progress) + m_repr.compute_progress(); + + return m_repr; + } + + void reset_fields_widths() + { + } + + const ProgressBarRepr& ProgressBar::repr() const + { + return m_repr; + } + + ProgressBarRepr& ProgressBar::repr() + { + return m_repr; } /********************** * DefaultProgressBar * **********************/ - DefaultProgressBar::DefaultProgressBar(const std::string& prefix, int width_cap) - : ProgressBar(prefix) - , m_progress(0) - , m_width_cap(width_cap) + DefaultProgressBar::DefaultProgressBar(const std::string& prefix, std::size_t total, int width) + : ProgressBar(prefix, total, width) { } - void DefaultProgressBar::print() + void DefaultProgressBar::print(std::ostream& ostream, std::size_t width, bool with_endl) { - std::cout << cursor::erase_line(2) << "\r"; - std::cout << m_prefix << "["; + if (!width && m_width) + width = m_width; - std::stringstream pf; - elapsed_time_to_stream(pf); - pf << m_postfix; - auto fpf = pf.str(); - int width = get_console_width(); - width = (width == -1) - ? m_width_cap - : (std::min)(static_cast(width - (m_prefix.size() + 4) - fpf.size()), - m_width_cap); - - if (!m_activate_bob) - { - ProgressScaleWriter w{ width, "=", ">", " " }; - w.write(std::cout, m_progress); - } - else - { - auto pos = static_cast(m_progress * width / 100.0); - for (int i = 0; i < width; ++i) - { - if (i == pos - 1) - { - std::cout << '<'; - } - else if (i == pos) - { - std::cout << '='; - } - else if (i == pos + 1) - { - std::cout << '>'; - } - else - { - std::cout << ' '; - } - } - } - std::cout << "] " << fpf; - } - - void DefaultProgressBar::set_full() - { - if (!m_start_time_saved) - { - set_start(); - } - m_activate_bob = false; - m_progress = 100; - } - - void DefaultProgressBar::set_progress(size_t current, size_t total) - { - if (!m_start_time_saved) - { - set_start(); - } - - if (current == SIZE_MAX) - { - m_activate_bob = true; - m_progress += 5; - } - else - { - size_t p = static_cast(current) / static_cast(total) * 100.; - m_activate_bob = false; - m_progress = p; - } - } - - void DefaultProgressBar::set_extracted() - { + print_formatted_bar_repr(ostream, m_repr, width, with_endl); } /********************* @@ -508,40 +1535,429 @@ namespace mamba HiddenProgressBar::HiddenProgressBar(const std::string& prefix, AggregatedBarManager* manager, - size_t total) - : ProgressBar(prefix) - , p_manager(manager) - , m_current(0) - , m_total(total) + std::size_t total, + int width) + : ProgressBar(prefix, total, width) { } - void HiddenProgressBar::print() + void HiddenProgressBar::print(std::ostream& /*stream*/, + std::size_t /*width*/, + bool /*with_endl*/) { } - void HiddenProgressBar::set_full() + /******************* + * MultiBarManager * + *******************/ + + MultiBarManager::MultiBarManager() { - if (!m_start_time_saved) + } + + MultiBarManager::MultiBarManager(std::size_t width) + : ProgressBarManager(width) + { + } + + ProgressProxy MultiBarManager::add_progress_bar(const std::string& name, + std::size_t expected_total) + { + std::string prefix = name; + std::lock_guard lock(m_mutex); + + m_progress_bars.push_back(std::make_unique(prefix, expected_total)); + return ProgressProxy(m_progress_bars[m_progress_bars.size() - 1].get()); + } + + std::size_t MultiBarManager::print(std::ostream& ostream, + std::size_t width, + std::size_t max_lines, + bool with_endl) + { + std::size_t active_count = 0, not_displayed = 0; + std::size_t max_sub_bars = std::numeric_limits::max(); + std::lock_guard lock(m_mutex); + + if (!width && m_width) + width = m_width; + + if (max_lines < std::numeric_limits::max()) + max_sub_bars = max_lines; + + std::vector displayed_bars = {}; { - set_start(); - } - p_manager->update_download_bar(m_total - m_current); - } + std::vector> pbar_locks = {}; - void HiddenProgressBar::set_progress(size_t current, size_t /*total*/) - { - if (!m_start_time_saved) + std::size_t max_bars_to_print = 0; + for (auto& pbar : m_progress_bars) + { + if (!pbar->stopped() && !pbar->completed()) + ++max_bars_to_print; + + pbar_locks.push_back(pbar->chrono_lock()); + } + + if (m_sort_bars) + sort_bars(max_bars_to_print <= max_sub_bars); + + for (auto& b : m_progress_bars) + { + if (b->started() || b->paused()) + { + if (active_count < max_sub_bars) + { + if (!b->started()) + b->repr().style = fmt::fg(fmt::terminal_color::bright_black); + else + b->repr().style = fmt::fg(fmt::terminal_color::white); + + displayed_bars.push_back(b.get()); + ++active_count; + } + else + ++not_displayed; + } + } + } + + if (!displayed_bars.empty()) { - set_start(); + compute_bars_progress(displayed_bars); + + if (max_sub_bars && active_count >= max_sub_bars) + { + ostream << fmt::format(" > {} more active", not_displayed) << "\n"; + ++active_count; + } + + for (std::size_t i = 0; i < displayed_bars.size(); ++i) + if ((i == displayed_bars.size() - 1) && !with_endl) + print_formatted_bar_repr(ostream, displayed_bars[i]->repr(), width, with_endl); + else + print_formatted_bar_repr(ostream, displayed_bars[i]->repr(), width, true); } - size_t old_current = m_current; - m_current = current; - p_manager->update_download_bar(m_current - old_current); + + return active_count; } - void HiddenProgressBar::set_extracted() + /************************ + * AggregatedBarManager * + ************************/ + + AggregatedBarManager::AggregatedBarManager() { - p_manager->update_extract_bar(); } -} + + AggregatedBarManager::AggregatedBarManager(std::size_t width) + : ProgressBarManager(width) + { + } + + ProgressProxy AggregatedBarManager::add_progress_bar(const std::string& prefix, + std::size_t expected_total) + { + std::lock_guard lock(m_mutex); + m_progress_bars.push_back( + std::make_unique(prefix, expected_total, 100)); + + return ProgressProxy(m_progress_bars[m_progress_bars.size() - 1].get()); + } + + void AggregatedBarManager::clear_progress_bars() + { + std::lock_guard lock(m_mutex); + m_labels.clear(); + m_progress_bars.clear(); + m_aggregated_bars.clear(); + } + + ProgressBar* AggregatedBarManager::aggregated_bar(const std::string& label) + { + std::lock_guard lock(m_mutex); + if (m_aggregated_bars.count(label)) + return m_aggregated_bars[label].get(); + else + return nullptr; + } + + void AggregatedBarManager::add_label(const std::string& label, + const ProgressProxy& progress_bar) + { + ProgressBarManager::add_label(label, progress_bar); + + std::lock_guard lock(m_mutex); + if (m_aggregated_bars.count(label) == 0) + m_aggregated_bars.insert({ label, + std::make_unique( + label, std::numeric_limits::max(), 100) }); + } + + void AggregatedBarManager::activate_sub_bars() + { + m_print_sub_bars = true; + } + + void AggregatedBarManager::deactivate_sub_bars() + { + m_print_sub_bars = false; + } + + void AggregatedBarManager::update_aggregates_progress() + { + for (auto& [label, bars] : m_labels) + { + std::size_t current = 0, total = 0, in_progress = 0, speed = 0, active_count = 0, + total_count = 0; + bool any_spinner = false; + bool any_started = false; + std::vector start_times = {}; + ProgressBar* aggregate_bar_ptr = m_aggregated_bars[label].get(); + + aggregate_bar_ptr->active_tasks().clear(); + aggregate_bar_ptr->all_tasks().clear(); + + for (auto& bar : bars) + { + current += bar->current(); + total += bar->total(); + ++total_count; + + if (!bar->unset()) + start_times.push_back(bar->start_time()); + if (bar->started()) + { + speed += bar->speed(); + in_progress += bar->total(); + ++active_count; + aggregate_bar_ptr->add_active_task(bar->prefix()); + any_started = true; + } + else + aggregate_bar_ptr->add_task(bar->prefix()); + + if (bar->is_spinner()) + any_spinner = true; + } + + if (aggregate_bar_ptr->unset() && !start_times.empty()) + aggregate_bar_ptr->start(*std::min_element(start_times.begin(), start_times.end())); + + if (any_spinner) + aggregate_bar_ptr->activate_spinner(); + else + aggregate_bar_ptr->deactivate_spinner(); + + if (any_started) + { + if (aggregate_bar_ptr->paused()) + aggregate_bar_ptr->resume(); + } + else + { + aggregate_bar_ptr->pause(); + aggregate_bar_ptr->deactivate_spinner(); + } + + if (any_started || (current != aggregate_bar_ptr->current()) + || (total != aggregate_bar_ptr->total())) + { + aggregate_bar_ptr->set_progress(current, total); + aggregate_bar_ptr->set_in_progress(in_progress); + aggregate_bar_ptr->set_speed(speed); + } + } + } + + std::size_t AggregatedBarManager::print(std::ostream& ostream, + std::size_t width, + std::size_t max_lines, + bool with_endl) + { + std::size_t active_count = 0, not_displayed = 0; + std::size_t max_sub_bars = std::numeric_limits::max(); + std::lock_guard lock(m_mutex); + + if (!width && m_width) + width = m_width; + + if (max_lines < std::numeric_limits::max()) + { + if (max_lines < m_labels.size()) + return 0; + else if (max_lines == m_labels.size()) + { + max_sub_bars = 0; + with_endl = false; + } + else + { + max_sub_bars = max_lines - m_labels.size(); + if (with_endl) + --max_sub_bars; + } + } + + std::vector displayed_bars = {}; + { + std::size_t max_bars_to_print = 0; + std::vector> pbar_locks = {}; + + for (auto& pbar : m_progress_bars) + { + if (!pbar->stopped() && !pbar->completed()) + ++max_bars_to_print; + + pbar_locks.push_back(pbar->chrono_lock()); + } + + if (m_sort_bars) + sort_bars(max_bars_to_print <= max_sub_bars); + + if (m_print_sub_bars) + { + for (auto& b : m_progress_bars) + { + if (b->started() || b->paused()) + { + if (active_count < max_sub_bars) + { + if (!b->started()) + b->repr().style = fmt::fg(fmt::terminal_color::bright_black); + else + b->repr().style = fmt::fg(fmt::terminal_color::white); + + displayed_bars.push_back(b.get()); + ++active_count; + } + else + ++not_displayed; + } + } + } + + update_aggregates_progress(); + for (auto& [label, b] : m_labels) + { + displayed_bars.push_back(m_aggregated_bars[label].get()); + ++active_count; + } + } + + if (!displayed_bars.empty()) + { + compute_bars_progress(displayed_bars); + + if (max_sub_bars && active_count > max_sub_bars) + { + ostream << fmt::format(" > {} more active", not_displayed) << "\n"; + ++active_count; + } + + for (std::size_t i = 0; i < displayed_bars.size(); ++i) + if ((i == displayed_bars.size() - 1) && !with_endl) + print_formatted_bar_repr(ostream, displayed_bars[i]->repr(), width, with_endl); + else + print_formatted_bar_repr(ostream, displayed_bars[i]->repr(), width, true); + } + + return active_count; + } + + void AggregatedBarManager::update_download_bar(std::size_t current_diff) + { + } + + void AggregatedBarManager::update_extract_bar() + { + } + + bool AggregatedBarManager::is_complete() const + { + return false; + } + + std::string duration_str(std::chrono::nanoseconds ns) + { + return duration_stream(ns).str(); + } + + std::stringstream duration_stream(std::chrono::nanoseconds ns) + { + using std::chrono::duration; + using std::chrono::duration_cast; + using std::chrono::hours; + using std::chrono::milliseconds; + using std::chrono::minutes; + using std::chrono::seconds; + + using days = duration>; + + std::stringstream sstream; + auto d = duration_cast(ns); + ns -= d; + auto h = duration_cast(ns); + ns -= h; + auto m = duration_cast(ns); + ns -= m; + auto s = duration_cast(ns); + ns -= s; + auto ms = duration_cast(ns); + int ms_rounded; + if (ms.count() >= 950) + { + ms_rounded = 0; + s += seconds(1); + } + else + ms_rounded = std::round(static_cast(ms.count()) / 100.); + + if (d.count() > 0) + sstream << d.count() << "d:"; + if (h.count() > 0) + sstream << h.count() << "h:"; + if (m.count() > 0) + sstream << m.count() << "m:"; + + sstream << s.count() << "." << ms_rounded << "s"; + return sstream; + } + + std::ostream& write_duration(std::ostream& os, std::chrono::nanoseconds ns) + { + os << duration_stream(ns).str(); + return os; + } + + int get_console_width() + { +#ifndef _WIN32 + struct winsize w; + ioctl(0, TIOCGWINSZ, &w); + return w.ws_col; +#else + + CONSOLE_SCREEN_BUFFER_INFO coninfo; + auto res = GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &coninfo); + return coninfo.dwSize.X; +#endif + + return -1; + } + + int get_console_height() + { +#ifndef _WIN32 + struct winsize w; + ioctl(0, TIOCGWINSZ, &w); + return w.ws_row; +#else + + CONSOLE_SCREEN_BUFFER_INFO coninfo; + auto res = GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &coninfo); + return coninfo.srWindow.Bottom - coninfo.srWindow.Top + 1; +#endif + + return -1; + } +} // namespace pbar diff --git a/libmamba/src/core/subdirdata.cpp b/libmamba/src/core/subdirdata.cpp index 2fc764bfb..0a4deccf3 100644 --- a/libmamba/src/core/subdirdata.cpp +++ b/libmamba/src/core/subdirdata.cpp @@ -68,7 +68,8 @@ namespace mamba const std::string& repodata_fn, MultiPackageCache& caches, bool is_noarch) - : m_loaded(false) + : m_progress_bar(ProgressProxy()) + , m_loaded(false) , m_download_complete(false) , m_repodata_url(repodata_url) , m_name(name) @@ -198,7 +199,7 @@ namespace mamba { std::string prefix = m_name; prefix.resize(PREFIX_LENGTH - 1, ' '); - Console::stream() << prefix << " Using cache"; + Console::stream() << fmt::format("{:<35} Using cache", prefix); } else { @@ -242,9 +243,13 @@ namespace mamba { LOG_INFO << "Unable to retrieve repodata (response: " << m_target->http_status << ") for '" << m_repodata_url << "'"; - m_progress_bar.set_postfix(std::to_string(m_target->http_status) + " Failed"); - m_progress_bar.set_full(); - m_progress_bar.mark_as_completed(); + + if (m_progress_bar) + { + m_progress_bar.set_postfix(std::to_string(m_target->http_status) + " failed"); + m_progress_bar.set_full(); + m_progress_bar.mark_as_completed(); + } m_loaded = false; return false; } @@ -314,7 +319,6 @@ namespace mamba m_valid_cache_path = writable_cache_path; } - { LOG_TRACE << "Refreshing '" << json_file.string() << "'"; auto lock = LockFile(json_file); @@ -328,9 +332,17 @@ namespace mamba m_solv_cache_valid = true; } - m_progress_bar.set_postfix("No change"); - m_progress_bar.set_full(); - m_progress_bar.mark_as_completed(); + if (m_progress_bar) + { + m_progress_bar.set_postfix("No change"); + m_progress_bar.mark_as_completed(); + + auto& r = m_progress_bar.repr(); + r.prefix.set_format("{:<35}"); + r.total.deactivate(); + r.speed.deactivate(); + r.elapsed.deactivate(); + } m_json_cache_valid = true; m_loaded = true; @@ -370,11 +382,13 @@ namespace mamba if (ends_with(m_repodata_url, ".bz2")) { - m_progress_bar.set_postfix("Decomp..."); + if (m_progress_bar) + m_progress_bar.set_postfix("decompressing"); decompress(); } - m_progress_bar.set_postfix("Finalizing..."); + if (m_progress_bar) + m_progress_bar.set_postfix("finalizing"); std::ifstream temp_file = open_ifstream(m_temp_file->path()); std::stringstream temp_json; @@ -397,9 +411,11 @@ namespace mamba exit(1); } - m_progress_bar.set_postfix("Done"); - m_progress_bar.set_full(); - m_progress_bar.mark_as_completed(); + if (m_progress_bar) + { + m_progress_bar.repr().postfix.set_value("downloaded").deactivate(); + m_progress_bar.mark_as_completed(); + } m_valid_cache_path = writable_cache_path; m_json_cache_valid = true; @@ -429,10 +445,14 @@ namespace mamba void MSubdirData::create_target(nlohmann::json& mod_etag) { + auto& ctx = Context::instance(); m_temp_file = std::make_unique(); - m_progress_bar = Console::instance().add_progress_bar(m_name); m_target = std::make_unique(m_name, m_repodata_url, m_temp_file->path()); - m_target->set_progress_bar(m_progress_bar); + if (!(ctx.no_progress_bars || ctx.quiet || ctx.json)) + { + m_progress_bar = Console::instance().add_progress_bar(m_name); + m_target->set_progress_bar(m_progress_bar); + } // if we get something _other_ than the noarch, we DO NOT throw if the file // can't be retrieved if (!m_is_noarch) diff --git a/libmamba/src/core/transaction.cpp b/libmamba/src/core/transaction.cpp index 77f4c4e58..fe6a1ac82 100644 --- a/libmamba/src/core/transaction.cpp +++ b/libmamba/src/core/transaction.cpp @@ -75,6 +75,9 @@ namespace mamba m_expected_size = pkg_info.size; m_sha256 = pkg_info.sha256; m_md5 = pkg_info.md5; + + auto& ctx = Context::instance(); + m_has_progress_bars = !(ctx.no_progress_bars || ctx.quiet || ctx.json); } void PackageDownloadExtractTarget::write_repodata_record(const fs::path& base_path) @@ -114,7 +117,12 @@ namespace mamba LOG_ERROR << "File not valid: file size doesn't match expectation " << m_tarball_path << "\nExpected: " << m_expected_size << "\nActual: " << size_t(m_target->downloaded_size) << "\n"; - m_progress_proxy.mark_as_completed("File size validation error."); + if (m_has_progress_bars) + { + m_download_bar.set_postfix("validation failed"); + m_download_bar.mark_as_completed(); + } + Console::instance().print(m_filename + " tarball has incorrect size"); m_validation_result = SIZE_ERROR; return; } @@ -126,7 +134,12 @@ namespace mamba if (m_sha256 != sha256sum) { m_validation_result = SHA256_ERROR; - m_progress_proxy.mark_as_completed("SHA256 sum validation error."); + if (m_has_progress_bars) + { + m_download_bar.set_postfix("validation failed"); + m_download_bar.mark_as_completed(); + } + Console::instance().print(m_filename + " tarball has incorrect checksum"); LOG_ERROR << "File not valid: SHA256 sum doesn't match expectation " << m_tarball_path << "\nExpected: " << m_sha256 << "\nActual: " << sha256sum << "\n"; @@ -139,23 +152,50 @@ namespace mamba if (m_md5 != md5sum) { m_validation_result = MD5SUM_ERROR; - m_progress_proxy.mark_as_completed("MD5 sum validation error."); + if (m_has_progress_bars) + { + m_download_bar.set_postfix("validation failed"); + m_download_bar.mark_as_completed(); + } + Console::instance().print(m_filename + " tarball has incorrect checksum"); LOG_ERROR << "File not valid: MD5 sum doesn't match expectation " << m_tarball_path << "\nExpected: " << m_md5 << "\nActual: " << md5sum << "\n"; } } } + std::function PackageDownloadExtractTarget::extract_repr() + { + return [&](ProgressBarRepr& r) -> void { + if (r.progress_bar().started()) + r.postfix.set_value("extracting"); + else + r.postfix.set_value("extracted"); + }; + } + + std::function PackageDownloadExtractTarget::extract_progress_callback() + { + return [&](ProgressProxy& bar) -> void { + if (bar.started()) + bar.set_progress(0, 1); + }; + } + bool PackageDownloadExtractTarget::extract() { // Extracting is __not__ yet thread safe it seems... interruption_point(); + + if (m_has_progress_bars) + m_extract_bar.start(); + LOG_DEBUG << "Waiting for decompression " << m_tarball_path; - m_progress_proxy.set_postfix("Waiting..."); + if (m_has_progress_bars) + m_extract_bar.update_progress(0, 1); { std::lock_guard lock(DownloadExtractSemaphore::semaphore); interruption_point(); - m_progress_proxy.set_postfix("Decompressing..."); LOG_DEBUG << "Decompressing '" << m_tarball_path.string() << "'"; fs::path extract_path; try @@ -185,69 +225,77 @@ namespace mamba LOG_DEBUG << "Extracted to '" << extract_path.string() << "'"; write_repodata_record(extract_path); add_url(); + + if (m_has_progress_bars) + { + m_extract_bar.set_full(); + m_extract_bar.mark_as_completed(); + } } catch (std::exception& e) { + Console::instance().print(m_filename + " extraction failed"); LOG_ERROR << "Error when extracting package: " << e.what(); m_decompress_exception = e; m_validation_result = VALIDATION_RESULT::EXTRACT_ERROR; - m_finished = true; - m_progress_proxy.mark_as_completed("Extraction error"); + if (m_has_progress_bars) + { + m_extract_bar.set_postfix("extraction failed"); + m_extract_bar.mark_as_completed(); + } return false; } } - m_finished = true; - return m_finished; + return true; } bool PackageDownloadExtractTarget::extract_from_cache() { - bool result = this->extract(); - if (result) - { - std::stringstream final_msg; - final_msg << "Extracted " << std::left << std::setw(30) << m_name; - m_progress_proxy.mark_as_completed(final_msg.str()); - return result; - } - // currently we always return true + this->extract(); + m_finished = true; return true; } bool PackageDownloadExtractTarget::validate_extract() { + using std::chrono::nanoseconds; + + if (m_has_progress_bars) + { + m_extract_bar.start(); + m_extract_bar.set_postfix("validating"); + } validate(); + // Validation if (m_validation_result != VALIDATION_RESULT::VALID) { + if (m_has_progress_bars) + m_extract_bar.set_postfix("validation failed"); + LOG_WARNING << "'" << m_tarball_path.string() << "' validation failed"; // abort here, but set finished to true m_finished = true; return true; } - std::stringstream final_msg; - final_msg << "Finished " << std::left << std::setw(30) << m_name << std::right - << std::setw(8); - m_progress_proxy.elapsed_time_to_stream(final_msg); - final_msg << " " << std::setw(12 + 2); - to_human_readable_filesize(final_msg, m_expected_size); - final_msg << " " << std::setw(6); - to_human_readable_filesize(final_msg, m_target->avg_speed); - final_msg << "/s"; - m_progress_proxy.mark_as_completed(final_msg.str()); + if (m_has_progress_bars) + m_extract_bar.set_postfix("validated"); + LOG_DEBUG << "'" << m_tarball_path.string() << "' successfully validated"; bool result = this->extract(); - m_progress_proxy.mark_as_extracted(); + m_finished = true; return result; } bool PackageDownloadExtractTarget::finalize_callback() { - m_progress_proxy.set_full(); - m_progress_proxy.set_postfix("Validating..."); + if (m_has_progress_bars) + { + m_download_bar.repr().postfix.set_value("downloaded").deactivate(); + m_download_bar.mark_as_completed(); + } LOG_INFO << "Download finished, validating '" << m_tarball_path.string() << "'"; - thread v(&PackageDownloadExtractTarget::validate_extract, this); v.detach(); @@ -279,6 +327,11 @@ namespace mamba return m_name; } + std::size_t PackageDownloadExtractTarget::expected_size() const + { + return m_expected_size; + } + // todo remove cache from this interface DownloadTarget* PackageDownloadExtractTarget::target(MultiPackageCache& caches) { @@ -296,15 +349,25 @@ namespace mamba caches.first_writable_cache(true).clear_query_cache(m_package_info); m_cache_path = caches.first_writable_path(); + if (m_has_progress_bars) + { + m_extract_bar = Console::instance().add_progress_bar(m_name, 1); + m_extract_bar.activate_spinner(); + m_extract_bar.set_progress_hook(extract_progress_callback()); + m_extract_bar.set_repr_hook(extract_repr()); + Console::instance().progress_bar_manager().add_label("Extract", m_extract_bar); + } + if (!tarball_cache.empty()) { LOG_DEBUG << "Found valid tarball cache at '" << tarball_cache.string() << "'"; m_tarball_path = tarball_cache / m_filename; - m_progress_proxy = Console::instance().add_progress_bar(m_name); m_validation_result = VALIDATION_RESULT::VALID; thread v(&PackageDownloadExtractTarget::extract_from_cache, this); v.detach(); + LOG_DEBUG << "Using cached tarball '" << m_filename << "'"; + return nullptr; } else { @@ -313,16 +376,21 @@ namespace mamba LOG_DEBUG << "Adding '" << m_name << "' to download targets from '" << m_url << "'"; m_tarball_path = m_cache_path / m_filename; - m_progress_proxy = Console::instance().add_progress_bar(m_name, m_expected_size); m_target = std::make_unique(m_name, m_url, m_tarball_path); m_target->set_finalize_callback(&PackageDownloadExtractTarget::finalize_callback, this); m_target->set_expected_size(m_expected_size); - m_target->set_progress_bar(m_progress_proxy); + if (m_has_progress_bars) + { + m_download_bar = Console::instance().add_progress_bar(m_name, m_expected_size); + m_target->set_progress_bar(m_download_bar); + Console::instance().progress_bar_manager().add_label("Download", + m_download_bar); + } return m_target.get(); } } - LOG_INFO << "Using cached '" << m_name << "'"; + LOG_DEBUG << "Using cached '" << m_name << "'"; m_finished = true; return nullptr; } @@ -350,8 +418,10 @@ namespace mamba MTransaction::MTransaction(MPool& pool, const std::vector& specs_to_remove, const std::vector& specs_to_install, - MultiPackageCache& caches) + MultiPackageCache& caches, + std::vector repos) : m_multi_cache(caches) + , m_repos(repos) { // auto& ctx = Context::instance(); std::vector pi_result; @@ -453,8 +523,11 @@ namespace mamba } - MTransaction::MTransaction(MSolver& solver, MultiPackageCache& caches) + MTransaction::MTransaction(MSolver& solver, + MultiPackageCache& caches, + std::vector repos) : m_multi_cache(caches) + , m_repos(repos) { if (!solver.is_solved()) { @@ -696,10 +769,12 @@ namespace mamba } LockFile lf(ctx.target_prefix / "conda-meta"); - clean_trash_files(ctx.target_prefix, false); Console::stream() << "\nTransaction starting"; + fetch_extract_packages(); + + // m_transaction_context = TransactionContext(prefix.path(), find_python_version()); History::UserRequest ur = History::UserRequest::prefilled(); TransactionRollback rollback; @@ -875,12 +950,14 @@ namespace mamba add_json(to_unlink, "UNLINK"); } - bool MTransaction::fetch_extract_packages(std::vector& repos) + bool MTransaction::fetch_extract_packages() { std::vector> targets; MultiDownloadTarget multi_dl; - Console::instance().init_multi_progress(ProgressBarMode::aggregated); + auto& pbar_manager + = Console::instance().init_progress_bar_manager(ProgressBarMode::aggregated); + auto& aggregated_pbar_manager = dynamic_cast(pbar_manager); auto& ctx = Context::instance(); DownloadExtractSemaphore::set_max(ctx.extract_threads); @@ -892,7 +969,7 @@ namespace mamba { std::string url; MRepo* mamba_repo = nullptr; - for (auto& r : repos) + for (auto& r : m_repos) { if (r->repo() == s->repo) { @@ -900,6 +977,7 @@ namespace mamba break; } } + if (mamba_repo == nullptr || mamba_repo->url() == "") { // use fallback mediadir / mediafile @@ -923,7 +1001,9 @@ namespace mamba } targets.emplace_back(std::make_unique(s)); - multi_dl.add(targets[targets.size() - 1]->target(m_multi_cache)); + DownloadTarget* download_target = targets.back()->target(m_multi_cache); + if (download_target != nullptr) + multi_dl.add(download_target); } if (ctx.experimental && ctx.verify_artifacts) @@ -932,9 +1012,75 @@ namespace mamba << "package(s) are trusted " << termcolor::reset; LOG_INFO << "All package(s) are trusted"; } - interruption_guard g([]() { Console::instance().init_multi_progress(); }); - bool downloaded = multi_dl.download(true); + if (!(ctx.no_progress_bars || ctx.json || ctx.quiet)) + { + interruption_guard g([]() { Console::instance().progress_bar_manager().terminate(); }); + + auto* dl_bar = aggregated_pbar_manager.aggregated_bar("Download"); + if (dl_bar) + dl_bar->set_repr_hook([&](ProgressBarRepr& repr) -> void { + auto active_tasks = dl_bar->active_tasks().size(); + if (active_tasks == 0) + { + repr.prefix.set_value(fmt::format("{:<16}", "Downloading")); + repr.postfix.set_value(fmt::format("{:<25}", "")); + } + else + { + repr.prefix.set_value(fmt::format( + "{:<11} {:>4}", "Downloading", fmt::format("({})", active_tasks))); + repr.postfix.set_value(fmt::format("{:<25}", dl_bar->last_active_task())); + } + repr.current.set_value( + fmt::format("{:>7}", to_human_readable_filesize(dl_bar->current(), 1))); + repr.separator.set_value("/"); + + std::string total_str; + if (dl_bar->total() == std::numeric_limits::max()) + total_str = "??.?MB"; + else + total_str = to_human_readable_filesize(dl_bar->total(), 1); + repr.total.set_value(fmt::format("{:>7}", total_str)); + + auto speed = dl_bar->avg_speed(std::chrono::milliseconds(500)); + repr.speed.set_value( + speed ? fmt::format("@ {:>7}/s", to_human_readable_filesize(speed, 1)) + : ""); + }); + + auto* extract_bar = aggregated_pbar_manager.aggregated_bar("Extract"); + if (extract_bar) + extract_bar->set_repr_hook([&](ProgressBarRepr& repr) -> void { + auto active_tasks = extract_bar->active_tasks().size(); + if (active_tasks == 0) + { + repr.prefix.set_value(fmt::format("{:<16}", "Extracting")); + repr.postfix.set_value(fmt::format("{:<25}", "")); + } + else + { + repr.prefix.set_value(fmt::format( + "{:<11} {:>4}", "Extracting", fmt::format("({})", active_tasks))); + repr.postfix.set_value( + fmt::format("{:<25}", extract_bar->last_active_task())); + } + repr.current.set_value(fmt::format("{:>3}", extract_bar->current())); + repr.separator.set_value("/"); + + std::string total_str; + if (extract_bar->total() == std::numeric_limits::max()) + total_str = "?"; + else + total_str = std::to_string(extract_bar->total()); + repr.total.set_value(fmt::format("{:>3}", total_str)); + }); + + pbar_manager.start(); + pbar_manager.watch_print(); + } + + bool downloaded = multi_dl.download(MAMBA_DOWNLOAD_FAILFAST | MAMBA_DOWNLOAD_SORT); bool all_valid = true; if (!downloaded) @@ -961,6 +1107,12 @@ namespace mamba std::this_thread::sleep_for(std::chrono::milliseconds(100)); } + if (!(ctx.no_progress_bars || ctx.json || ctx.quiet)) + { + pbar_manager.terminate(); + pbar_manager.clear_progress_bars(); + } + for (const auto& t : targets) { if (t->validation_result() != PackageDownloadExtractTarget::VALIDATION_RESULT::VALID @@ -982,20 +1134,13 @@ namespace mamba return m_to_install.size() == 0 && m_to_remove.size() == 0; } - bool MTransaction::prompt(std::vector& repos) + bool MTransaction::prompt() { print(); if (Context::instance().dry_run || empty()) - { return true; - } - bool res = Console::prompt("Confirm changes", 'y'); - if (res) - { - return fetch_extract_packages(repos); - } - return res; + return Console::prompt("Confirm changes", 'y'); } void MTransaction::print() @@ -1241,7 +1386,8 @@ namespace mamba MTransaction create_explicit_transaction_from_urls(MPool& pool, const std::vector& urls, - MultiPackageCache& package_caches) + MultiPackageCache& package_caches, + std::vector& repos) { std::vector specs_to_install; for (auto& u : urls) @@ -1267,6 +1413,6 @@ namespace mamba } specs_to_install.push_back(ms); } - return MTransaction(pool, {}, specs_to_install, package_caches); + return MTransaction(pool, {}, specs_to_install, package_caches, repos); } } // namespace mamba diff --git a/libmamba/src/core/util.cpp b/libmamba/src/core/util.cpp index 84cdfd117..3ae64bc4c 100644 --- a/libmamba/src/core/util.cpp +++ b/libmamba/src/core/util.cpp @@ -74,18 +74,6 @@ namespace mamba return status.type() != fs::file_type::not_found || status.type() == fs::file_type::symlink; } - void to_human_readable_filesize(std::ostream& o, double bytes, std::size_t precision) - { - const char* sizes[] = { " B", " KB", " MB", " GB", " TB" }; - int order = 0; - while (bytes >= 1024 && order < (5 - 1)) - { - order++; - bytes = bytes / 1024; - } - o << std::fixed << std::setprecision(precision) << bytes << sizes[order]; - } - std::vector filter_dir(const fs::path& dir, const std::string& suffix) { std::vector result; diff --git a/libmamba/tests/CMakeLists.txt b/libmamba/tests/CMakeLists.txt index 542a0aa9e..3412e0fa7 100644 --- a/libmamba/tests/CMakeLists.txt +++ b/libmamba/tests/CMakeLists.txt @@ -6,23 +6,25 @@ find_package(Threads REQUIRED) include_directories(${GTEST_INCLUDE_DIRS} SYSTEM) set(TEST_SRCS + test_activation.cpp test_channel.cpp test_configuration.cpp test_cpp.cpp - test_url.cpp - history_test/test_history.cpp - test_shell_init.cpp - test_activation.cpp - test_string_methods.cpp + test_env_file_reading.cpp test_environments_manager.cpp - test_transfer.cpp - test_thread_utils.cpp test_graph.cpp + history_test/test_history.cpp + test_lockfile.cpp test_pinning.cpp + test_output.cpp + test_progress_bar.cpp + test_shell_init.cpp + test_string_methods.cpp + test_thread_utils.cpp + test_transfer.cpp + test_url.cpp test_validate.cpp test_virtual_packages.cpp - test_env_file_reading.cpp - test_lockfile.cpp ) message(STATUS "Building libmamba C++ tests") diff --git a/libmamba/tests/test_cpp.cpp b/libmamba/tests/test_cpp.cpp index b8a2f5ca9..0df11276b 100644 --- a/libmamba/tests/test_cpp.cpp +++ b/libmamba/tests/test_cpp.cpp @@ -255,19 +255,6 @@ namespace mamba "http://root:*****@myweb.com/test.repo\nhttp://myweb.com/t/*****/test.repo http://myweb.com/t/*****/test.repo http://root:*****@myweb.com/test.repo"); } - TEST(output, no_progress_bars) - { - Context::instance().no_progress_bars = true; - testing::internal::CaptureStdout(); - - auto proxy = Console::instance().add_progress_bar("conda-forge"); - proxy.set_progress(50, 100); - proxy.set_postfix("Downloading"); - proxy.mark_as_completed("conda-forge channel downloaded"); - std::string output = testing::internal::GetCapturedStdout(); - EXPECT_TRUE(ends_with(output, "conda-forge channel downloaded\n")); - Context::instance().no_progress_bars = false; - } class OutputPromptTests : public testing::TestWithParam> { diff --git a/libmamba/tests/test_output.cpp b/libmamba/tests/test_output.cpp new file mode 100644 index 000000000..94044b7b9 --- /dev/null +++ b/libmamba/tests/test_output.cpp @@ -0,0 +1,24 @@ + +#include + +#include +#include + +#include "mamba/core/context.hpp" +#include "mamba/core/output.hpp" + +namespace mamba +{ + TEST(output, no_progress_bars) + { + Context::instance().no_progress_bars = true; + auto proxy = Console::instance().add_progress_bar("conda-forge"); + EXPECT_FALSE(proxy.defined()); + EXPECT_FALSE(proxy); + + Context::instance().no_progress_bars = false; + proxy = Console::instance().add_progress_bar("conda-forge"); + EXPECT_TRUE(proxy.defined()); + EXPECT_TRUE(proxy); + } +} // namespace mamba diff --git a/libmamba/tests/test_progress_bar.cpp b/libmamba/tests/test_progress_bar.cpp new file mode 100644 index 000000000..35769f877 --- /dev/null +++ b/libmamba/tests/test_progress_bar.cpp @@ -0,0 +1,381 @@ +#include + +#include +#include + +#include "mamba/core/progress_bar.hpp" + + +namespace mamba +{ + class progress_bar : public ::testing::Test + { + public: + progress_bar() + { + p_progress_bar_manager = std::make_unique(); + proxy = p_progress_bar_manager->add_progress_bar("conda-forge"); + + auto& r = proxy.repr(); + r.progress.set_value("??"); + r.current.set_value("foo"); + r.separator.set_value("-"); + r.total.set_value("bar"); + r.speed.set_value("@10"); + r.postfix.set_value("downloading"); + r.elapsed.set_value("0.1s"); + } + + protected: + std::unique_ptr p_progress_bar_manager; + ProgressProxy proxy; + std::ostringstream ostream; + }; + + TEST_F(progress_bar, print) + { + auto& r = proxy.repr(); + + EXPECT_TRUE(r.prefix.active()); + EXPECT_EQ(r.prefix.value(), "conda-forge"); + EXPECT_EQ(r.prefix.width(), 11); + + EXPECT_TRUE(r.progress); + EXPECT_EQ(r.progress.value(), "??"); + EXPECT_EQ(r.progress.width(), 2); + + EXPECT_TRUE(r.separator); + EXPECT_EQ(r.separator.value(), "-"); + EXPECT_EQ(r.separator.width(), 1); + + EXPECT_TRUE(r.total); + EXPECT_EQ(r.total.value(), "bar"); + EXPECT_EQ(r.total.width(), 3); + + EXPECT_TRUE(r.speed); + EXPECT_EQ(r.speed.value(), "@10"); + EXPECT_EQ(r.speed.width(), 3); + + EXPECT_TRUE(r.postfix.active()); + EXPECT_EQ(r.postfix.value(), "downloading"); + EXPECT_EQ(r.postfix.width(), 11); + + EXPECT_TRUE(r.elapsed.active()); + EXPECT_EQ(r.elapsed.value(), "0.1s"); + EXPECT_EQ(r.elapsed.width(), 4); + + proxy.print(ostream, 0, false); + EXPECT_EQ(ostream.str(), "conda-forge ?? foo - bar @10 downloading 0.1s"); + ostream.str(""); + + r.set_width(21); // no impact if 'update_repr' not called + proxy.print(ostream, 0, false); + EXPECT_EQ(ostream.str(), "conda-forge ?? foo - bar @10 downloading 0.1s"); + ostream.str(""); + } + + TEST_F(progress_bar, print_no_resize) + { + auto& r = proxy.repr(); + + r.set_width(150); + proxy.update_repr(); + EXPECT_TRUE(r.prefix); + EXPECT_TRUE(r.progress); + EXPECT_TRUE(r.current); + EXPECT_TRUE(r.separator); + EXPECT_TRUE(r.total); + EXPECT_TRUE(r.speed); + EXPECT_TRUE(r.postfix); + EXPECT_TRUE(r.elapsed); + EXPECT_EQ(r.prefix.width(), 11); + EXPECT_EQ(r.progress.width(), 106); + EXPECT_EQ(r.current.width(), 3); + EXPECT_EQ(r.separator.width(), 1); + EXPECT_EQ(r.total.width(), 3); + EXPECT_EQ(r.speed.width(), 3); + EXPECT_EQ(r.postfix.width(), 11); + EXPECT_EQ(r.elapsed.width(), 5); + } + + TEST_F(progress_bar, print_reduce_bar) + { + auto& r = proxy.repr(); + + r.set_width(84); + proxy.update_repr(); + EXPECT_TRUE(r.prefix); + EXPECT_TRUE(r.progress); + EXPECT_TRUE(r.current); + EXPECT_TRUE(r.separator); + EXPECT_TRUE(r.total); + EXPECT_TRUE(r.speed); + EXPECT_TRUE(r.postfix); + EXPECT_TRUE(r.elapsed); + EXPECT_EQ(r.prefix.width(), 11); + EXPECT_EQ(r.progress.width(), 40); + EXPECT_EQ(r.current.width(), 3); + EXPECT_EQ(r.separator.width(), 1); + EXPECT_EQ(r.total.width(), 3); + EXPECT_EQ(r.speed.width(), 3); + EXPECT_EQ(r.postfix.width(), 11); + EXPECT_EQ(r.elapsed.width(), 5); + + // 1: reduce bar width + // available space redistributed to the bar + r.set_width(83); + proxy.update_repr(); + EXPECT_TRUE(r.prefix); + EXPECT_TRUE(r.progress); + EXPECT_TRUE(r.current); + EXPECT_TRUE(r.separator); + EXPECT_TRUE(r.total); + EXPECT_TRUE(r.speed); + EXPECT_TRUE(r.postfix); + EXPECT_TRUE(r.elapsed); + EXPECT_EQ(r.prefix.width(), 11); + EXPECT_EQ(r.progress.width(), 39); + EXPECT_EQ(r.current.width(), 3); + EXPECT_EQ(r.separator.width(), 1); + EXPECT_EQ(r.total.width(), 3); + EXPECT_EQ(r.speed.width(), 3); + EXPECT_EQ(r.postfix.width(), 11); + EXPECT_EQ(r.elapsed.width(), 5); + } + + TEST_F(progress_bar, print_remove_total_sep) + { + auto& r = proxy.repr(); + + r.set_width(59); + proxy.update_repr(); + EXPECT_TRUE(r.prefix); + EXPECT_TRUE(r.progress); + EXPECT_TRUE(r.current); + EXPECT_TRUE(r.separator); + EXPECT_TRUE(r.total); + EXPECT_TRUE(r.speed); + EXPECT_TRUE(r.postfix); + EXPECT_TRUE(r.elapsed); + EXPECT_EQ(r.prefix.width(), 11); + EXPECT_EQ(r.progress.width(), 15); + EXPECT_EQ(r.current.width(), 3); + EXPECT_EQ(r.separator.width(), 1); + EXPECT_EQ(r.total.width(), 3); + EXPECT_EQ(r.speed.width(), 3); + EXPECT_EQ(r.postfix.width(), 11); + EXPECT_EQ(r.elapsed.width(), 5); + + // 2: remove the total value and the separator + // available space redistributed to the bar + r.set_width(58); + proxy.update_repr(); + EXPECT_TRUE(r.prefix); + EXPECT_TRUE(r.progress); + EXPECT_TRUE(r.current); + EXPECT_FALSE(r.separator); + EXPECT_FALSE(r.total); + EXPECT_TRUE(r.speed); + EXPECT_TRUE(r.postfix); + EXPECT_TRUE(r.elapsed); + EXPECT_EQ(r.prefix.width(), 11); + EXPECT_EQ(r.progress.width(), 20); + EXPECT_EQ(r.current.width(), 3); + EXPECT_EQ(r.speed.width(), 3); + EXPECT_EQ(r.postfix.width(), 11); + EXPECT_EQ(r.elapsed.width(), 5); + } + + TEST_F(progress_bar, print_remove_speed) + { + auto& r = proxy.repr(); + + r.set_width(53); + proxy.update_repr(); + EXPECT_TRUE(r.prefix); + EXPECT_TRUE(r.progress); + EXPECT_TRUE(r.current); + EXPECT_FALSE(r.separator); + EXPECT_FALSE(r.total); + EXPECT_TRUE(r.speed); + EXPECT_TRUE(r.postfix); + EXPECT_TRUE(r.elapsed); + EXPECT_EQ(r.prefix.width(), 11); + EXPECT_EQ(r.progress.width(), 15); + EXPECT_EQ(r.current.width(), 3); + EXPECT_EQ(r.speed.width(), 3); + EXPECT_EQ(r.postfix.width(), 11); + EXPECT_EQ(r.elapsed.width(), 5); + + // 3: remove the speed + // available space redistributed to the bar + r.set_width(52); + proxy.update_repr(); + EXPECT_TRUE(r.prefix); + EXPECT_TRUE(r.progress); + EXPECT_TRUE(r.current); + EXPECT_FALSE(r.separator); + EXPECT_FALSE(r.total); + EXPECT_FALSE(r.speed); + EXPECT_TRUE(r.postfix); + EXPECT_TRUE(r.elapsed); + EXPECT_EQ(r.prefix.width(), 11); + EXPECT_EQ(r.progress.width(), 18); + EXPECT_EQ(r.current.width(), 3); + EXPECT_EQ(r.postfix.width(), 11); + EXPECT_EQ(r.elapsed.width(), 5); + } + + TEST_F(progress_bar, print_remove_postfix) + { + auto& r = proxy.repr(); + + r.set_width(49); + proxy.update_repr(); + EXPECT_TRUE(r.prefix); + EXPECT_TRUE(r.progress); + EXPECT_TRUE(r.current); + EXPECT_FALSE(r.separator); + EXPECT_FALSE(r.total); + EXPECT_FALSE(r.speed); + EXPECT_TRUE(r.postfix); + EXPECT_TRUE(r.elapsed); + EXPECT_EQ(r.prefix.width(), 11); + EXPECT_EQ(r.progress.width(), 15); + EXPECT_EQ(r.current.width(), 3); + EXPECT_EQ(r.postfix.width(), 11); + EXPECT_EQ(r.elapsed.width(), 5); + + // 4: remove the postfix + // available space redistributed to the bar + r.set_width(48); + proxy.update_repr(); + EXPECT_TRUE(r.prefix); + EXPECT_TRUE(r.progress); + EXPECT_TRUE(r.current); + EXPECT_FALSE(r.separator); + EXPECT_FALSE(r.total); + EXPECT_FALSE(r.speed); + EXPECT_FALSE(r.postfix); + EXPECT_TRUE(r.elapsed); + EXPECT_EQ(r.prefix.width(), 11); + EXPECT_EQ(r.progress.width(), 26); + EXPECT_EQ(r.current.width(), 3); + EXPECT_EQ(r.elapsed.width(), 5); + } + + TEST_F(progress_bar, print_truncate_prefix) + { + auto& r = proxy.repr(); + proxy.set_prefix("some_very_very_long_prefix"); + + r.set_width(52); + proxy.update_repr(); + EXPECT_TRUE(r.prefix); + EXPECT_TRUE(r.progress); + EXPECT_TRUE(r.current); + EXPECT_FALSE(r.separator); + EXPECT_FALSE(r.total); + EXPECT_FALSE(r.speed); + EXPECT_FALSE(r.postfix); + EXPECT_TRUE(r.elapsed); + EXPECT_EQ(r.prefix.width(), 26); + EXPECT_EQ(r.progress.width(), 15); + EXPECT_EQ(r.current.width(), 3); + EXPECT_EQ(r.elapsed.width(), 5); + + // 5: truncate the prefix if too long + // available space redistributed to the prefix + r.set_width(51); + proxy.update_repr(); + EXPECT_TRUE(r.prefix); + EXPECT_TRUE(r.progress); + EXPECT_TRUE(r.current); + EXPECT_FALSE(r.separator); + EXPECT_FALSE(r.total); + EXPECT_FALSE(r.speed); + EXPECT_FALSE(r.postfix); + EXPECT_TRUE(r.elapsed); + EXPECT_EQ(r.prefix.width(), 25); + EXPECT_EQ(r.progress.width(), 15); + EXPECT_EQ(r.current.width(), 3); + EXPECT_EQ(r.elapsed.width(), 5); + } + + TEST_F(progress_bar, print_without_bar) + { + auto& r = proxy.repr(); + + r.set_width(34).reset_fields(); + proxy.update_repr(); + EXPECT_TRUE(r.prefix); + EXPECT_TRUE(r.progress); + EXPECT_TRUE(r.current); + EXPECT_FALSE(r.separator); + EXPECT_FALSE(r.total); + EXPECT_FALSE(r.speed); + EXPECT_FALSE(r.postfix); + EXPECT_TRUE(r.elapsed); + EXPECT_EQ(r.prefix.width(), 11); + EXPECT_EQ(r.progress.width(), 12); + EXPECT_EQ(r.current.width(), 3); + EXPECT_TRUE(r.progress.overflow()); + EXPECT_EQ(r.elapsed.width(), 5); + + // 6: display progress without a bar + r.set_width(33); + proxy.update_repr(); + proxy.print(ostream, 0, false); + EXPECT_EQ(ostream.str(), "conda-forge 0% foo --"); + ostream.str(""); + } + + TEST_F(progress_bar, print_remove_current) + { + auto& r = proxy.repr(); + + r.set_width(26).reset_fields(); + proxy.update_repr(); + proxy.print(ostream, 0, false); + EXPECT_EQ(ostream.str(), "conda-forge 0% foo --"); + ostream.str(""); + + // 7: remove the current value + r.set_width(25).reset_fields(); + proxy.update_repr(); + proxy.print(ostream, 0, false); + EXPECT_EQ(ostream.str(), "conda-forge 0% --"); + ostream.str(""); + } + + TEST_F(progress_bar, print_remove_elapsed) + { + auto& r = proxy.repr(); + + r.set_width(22).reset_fields(); + proxy.update_repr(); + EXPECT_TRUE(r.prefix); + EXPECT_TRUE(r.progress); + EXPECT_FALSE(r.current); + EXPECT_FALSE(r.separator); + EXPECT_FALSE(r.total); + EXPECT_FALSE(r.speed); + EXPECT_FALSE(r.postfix); + EXPECT_TRUE(r.elapsed); + proxy.print(ostream, 0, false); + EXPECT_EQ(r.prefix.width(), 11); + EXPECT_EQ(r.progress.width(), 4); + EXPECT_EQ(r.elapsed.width(), 5); + EXPECT_EQ(ostream.str(), "conda-forge 0% --"); + ostream.str(""); + + // 8: remove the elapsed time + r.set_width(21); + proxy.update_repr(); + proxy.print(ostream, 0, false); + EXPECT_EQ(r.prefix.width(), 11); + EXPECT_EQ(r.progress.width(), 9); + EXPECT_EQ(ostream.str(), "conda-forge 0%"); + ostream.str(""); + } +} // namespace mamba diff --git a/libmamba/tests/test_thread_utils.cpp b/libmamba/tests/test_thread_utils.cpp index 066196d24..22fc64ac1 100644 --- a/libmamba/tests/test_thread_utils.cpp +++ b/libmamba/tests/test_thread_utils.cpp @@ -17,12 +17,12 @@ namespace mamba // Ensures the compiler doe snot optimize away Context::instance() std::string current_command = Context::instance().current_command; EXPECT_EQ(current_command, "mamba"); - Console::instance().init_multi_progress(); + Console::instance().init_progress_bar_manager(ProgressBarMode::multi); { interruption_guard g([&res]() { // Test for double free (segfault if that happens) std::cout << "Interruption guard is interrupting" << std::endl; - Console::instance().init_multi_progress(); + Console::instance().init_progress_bar_manager(ProgressBarMode::multi); { std::unique_lock lk(res_mutex); res -= 100; diff --git a/libmamba/tests/test_transfer.cpp b/libmamba/tests/test_transfer.cpp index d814f86a9..2d058ab52 100644 --- a/libmamba/tests/test_transfer.cpp +++ b/libmamba/tests/test_transfer.cpp @@ -23,7 +23,7 @@ namespace mamba // file:// url should not retry EXPECT_EQ(cf.target()->can_retry(), false); - multi_dl.download(true); + multi_dl.download(MAMBA_DOWNLOAD_FAILFAST); // File does not exist EXPECT_EQ(cf.target()->result, 37); @@ -38,7 +38,7 @@ namespace mamba true); cf.load(); multi_dl.add(cf.target()); - EXPECT_THROW(multi_dl.download(true), std::runtime_error); + EXPECT_THROW(multi_dl.download(MAMBA_DOWNLOAD_FAILFAST), std::runtime_error); } Context::instance().quiet = false; #endif diff --git a/libmambapy/src/main.cpp b/libmambapy/src/main.cpp index 18835fa88..51d6b1273 100644 --- a/libmambapy/src/main.cpp +++ b/libmambapy/src/main.cpp @@ -82,16 +82,14 @@ PYBIND11_MODULE(bindings, m) .def("clear", &MRepo::clear); py::class_(m, "Transaction") - .def(py::init()) + .def(py::init&>()) .def("to_conda", &MTransaction::to_conda) .def("log_json", &MTransaction::log_json) .def("print", &MTransaction::print) .def("fetch_extract_packages", &MTransaction::fetch_extract_packages) .def("prompt", &MTransaction::prompt) .def("find_python_version", &MTransaction::find_python_version) - .def("execute", [](MTransaction& self, PrefixData& target_prefix) -> bool { - return self.execute(target_prefix); - }); + .def("execute", &MTransaction::execute); py::class_(m, "Solver") .def(py::init>>()) @@ -505,6 +503,10 @@ PYBIND11_MODULE(bindings, m) m.attr("MAMBA_ONLY_DEPS") = MAMBA_ONLY_DEPS; m.attr("MAMBA_FORCE_REINSTALL") = MAMBA_FORCE_REINSTALL; + // DOWNLOAD FLAGS + m.attr("MAMBA_DOWNLOAD_FAILFAST") = MAMBA_DOWNLOAD_FAILFAST; + m.attr("MAMBA_DOWNLOAD_SORT") = MAMBA_DOWNLOAD_SORT; + // CLEAN FLAGS m.attr("MAMBA_CLEAN_ALL") = MAMBA_CLEAN_ALL; m.attr("MAMBA_CLEAN_INDEX") = MAMBA_CLEAN_INDEX; diff --git a/mamba/mamba/mamba.py b/mamba/mamba/mamba.py index 427fab58b..e785cf229 100644 --- a/mamba/mamba/mamba.py +++ b/mamba/mamba/mamba.py @@ -8,6 +8,7 @@ import os import sys from logging import getLogger from os.path import isdir, isfile, join +from pathlib import Path # create support from conda.base.constants import ChannelPriority, DepsModifier, UpdateModifier @@ -217,10 +218,12 @@ def remove(args, parser): return exit_code package_cache = api.MultiPackageCache(context.pkgs_dirs) - transaction = api.Transaction(solver, package_cache) - downloaded = transaction.prompt(repos) - if not downloaded: + transaction = api.Transaction(solver, package_cache, repos) + + if not transaction.prompt(): exit(0) + elif not context.dry_run: + transaction.fetch_extract_packages() mmb_specs, to_link, to_unlink = transaction.to_conda() transaction.log_json() @@ -552,23 +555,30 @@ def install(args, parser, command="install"): return exit_code package_cache = api.MultiPackageCache(context.pkgs_dirs) - transaction = api.Transaction(solver, package_cache) + transaction = api.Transaction(solver, package_cache, repos) mmb_specs, to_link, to_unlink = transaction.to_conda() specs_to_add = [MatchSpec(m) for m in mmb_specs[0]] specs_to_remove = [MatchSpec(m) for m in mmb_specs[1]] transaction.log_json() - downloaded = transaction.prompt(repos) - if not downloaded: - exit(0) - # if use_mamba_experimental and not os.name == "nt": if use_mamba_experimental: - if newenv and not isdir(context.target_prefix) and not context.dry_run: - mkdir_p(prefix) - transaction.execute(prefix_data) + if transaction.prompt(): + if ( + newenv + and not isdir(Path(prefix) / "conda-meta") + and not context.dry_run + ): + mkdir_p(Path(prefix) / "conda-meta") + + transaction.execute(prefix_data) else: + if not transaction.prompt(): + exit(0) + elif not context.dry_run: + transaction.fetch_extract_packages() + conda_transaction = to_txn( specs_to_add, specs_to_remove, diff --git a/mamba/mamba/mamba_env.py b/mamba/mamba/mamba_env.py index d5ffb1f0e..bfaa5fe4f 100644 --- a/mamba/mamba/mamba_env.py +++ b/mamba/mamba/mamba_env.py @@ -136,15 +136,17 @@ def mamba_install(prefix, specs, args, env, dry_run=False, *_, **kwargs): exit(1) package_cache = api.MultiPackageCache(context.pkgs_dirs) - transaction = api.Transaction(solver, package_cache) + transaction = api.Transaction(solver, package_cache, repos) mmb_specs, to_link, to_unlink = transaction.to_conda() specs_to_add = [MatchSpec(m) for m in mmb_specs[0]] transaction.log_json() - downloaded = transaction.prompt(repos) - if not downloaded: + if not transaction.prompt(): exit(0) + elif not context.dry_run: + transaction.fetch_extract_packages() + if prune: history = api.History(prefix) history_map = history.get_requested_specs_map() @@ -162,7 +164,9 @@ def mamba_install(prefix, specs, args, env, dry_run=False, *_, **kwargs): conda_transaction = to_txn( specs_to_add, [], prefix, to_link, to_unlink, installed_pkg_recs, index ) + handle_txn(conda_transaction, prefix, args, True) + try: installed_json_f.close() os.unlink(installed_json_f.name) diff --git a/mamba/mamba/utils.py b/mamba/mamba/utils.py index 5dd74339c..379e4e631 100644 --- a/mamba/mamba/utils.py +++ b/mamba/mamba/utils.py @@ -100,7 +100,7 @@ def get_index( ) dlist.add(sd) - is_downloaded = dlist.download(True) + is_downloaded = dlist.download(api.MAMBA_DOWNLOAD_FAILFAST) if not is_downloaded: raise RuntimeError("Error downloading repodata.") diff --git a/micromamba/tests/helpers.py b/micromamba/tests/helpers.py index 184f0dd7a..da2c213a4 100644 --- a/micromamba/tests/helpers.py +++ b/micromamba/tests/helpers.py @@ -115,8 +115,7 @@ def install(*args, default_channel=True, no_rc=True, no_dry_run=False): cmd += ["--dry-run"] cmd += ["--log-level=info"] - print(f"Running command {' '.join(cmd)}", file=sys.stderr) - res = subprocess.check_output(cmd, stderr=sys.stderr) + res = subprocess.check_output(cmd) if "--json" in args: try: