mirror of https://github.com/mamba-org/mamba.git
maint: Cleanup CMake files and delete not compiled files (#3667)
This commit is contained in:
Ayaz Salikhov
GitHub
parent
ddc3d40b35
commit
76d06a7f5a
|
|
@ -162,8 +162,8 @@ set(
|
|||
${LIBMAMBA_SOURCE_DIR}/util/path_manip.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/util/random.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/util/string.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/util/url.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/util/url_manip.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/util/url.cpp
|
||||
# Implementation of version and matching specs
|
||||
${LIBMAMBA_SOURCE_DIR}/specs/archive.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/specs/authentication_info.cpp
|
||||
|
|
@ -178,8 +178,8 @@ set(
|
|||
${LIBMAMBA_SOURCE_DIR}/specs/regex_spec.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/specs/repo_data.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/specs/unresolved_channel.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/specs/version.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/specs/version_spec.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/specs/version.cpp
|
||||
# Solver generic interface
|
||||
${LIBMAMBA_SOURCE_DIR}/solver/helpers.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/solver/problems_graph.cpp
|
||||
|
|
@ -192,61 +192,61 @@ set(
|
|||
${LIBMAMBA_SOURCE_DIR}/solver/libsolv/solver.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/solver/libsolv/unsolvable.cpp
|
||||
# Artifacts validation
|
||||
${LIBMAMBA_SOURCE_DIR}/validation/tools.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/validation/errors.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/validation/keys.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/validation/update_framework.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/validation/repo_checker.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/validation/tools.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/validation/update_framework_v0_6.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/validation/update_framework_v1.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/validation/repo_checker.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/validation/update_framework.cpp
|
||||
# Downloaders and mirrors
|
||||
${LIBMAMBA_SOURCE_DIR}/download/compression.hpp
|
||||
${LIBMAMBA_SOURCE_DIR}/download/compression.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/download/curl.hpp
|
||||
${LIBMAMBA_SOURCE_DIR}/download/compression.hpp
|
||||
${LIBMAMBA_SOURCE_DIR}/download/curl.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/download/curl.hpp
|
||||
${LIBMAMBA_SOURCE_DIR}/download/downloader_impl.hpp
|
||||
${LIBMAMBA_SOURCE_DIR}/download/downloader.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/download/mirror.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/download/mirror_map.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/download/mirror_impl.hpp
|
||||
${LIBMAMBA_SOURCE_DIR}/download/mirror_impl.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/download/mirror_impl.hpp
|
||||
${LIBMAMBA_SOURCE_DIR}/download/mirror_map.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/download/mirror.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/download/request.cpp
|
||||
# Core API (low-level)
|
||||
${LIBMAMBA_SOURCE_DIR}/core/singletons.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/activation.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/package_database_loader.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/channel_context.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/context.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/download_progress_bar.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/env_lockfile.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/environments_manager.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/error_handling.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/transaction_context.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/link.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/execution.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/fsutil.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/history.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/link.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/menuinst.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/output.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/package_handling.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/package_cache.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/prefix_data.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/progress_bar.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/progress_bar_impl.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/pinning.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/package_database_loader.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/package_fetcher.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/package_handling.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/package_paths.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/pinning.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/prefix_data.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/progress_bar_impl.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/progress_bar.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/query.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/repo_checker_store.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/run.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/shell_init.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/singletons.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/subdirdata.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/thread_utils.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/transaction.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/util.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/fsutil.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/util_os.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/virtual_packages.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/env_lockfile.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/execution.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/timeref.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/transaction_context.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/transaction.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/util_os.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/util.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/core/virtual_packages.cpp
|
||||
# API (high-level)
|
||||
${LIBMAMBA_SOURCE_DIR}/api/c_api.cpp
|
||||
${LIBMAMBA_SOURCE_DIR}/api/channel_loader.cpp
|
||||
|
|
@ -301,19 +301,19 @@ set(
|
|||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/iterator.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/json.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/loop_control.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/os.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/os_linux.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/os_osx.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/os_unix.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/os_win.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/os.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/parsers.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/path_manip.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/random.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/string.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/tuple_hash.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/type_traits.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/url.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/url_manip.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/url.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/variant_cmp.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/util/weakening_map.hpp
|
||||
# Implementation of version and matching specs
|
||||
|
|
@ -321,16 +321,18 @@ set(
|
|||
${LIBMAMBA_INCLUDE_DIR}/mamba/specs/authentication_info.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/specs/build_number_spec.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/specs/channel.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/specs/chimera_string_spec.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/specs/conda_url.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/specs/error.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/specs/glob_spec.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/specs/match_spec.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/specs/package_info.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/specs/platform.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/specs/regex_spec.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/specs/repo_data.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/specs/unresolved_channel.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/specs/version.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/specs/version_spec.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/specs/version.hpp
|
||||
# Solver generic interface
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/solver/problems_graph.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/solver/request.hpp
|
||||
|
|
@ -342,63 +344,64 @@ set(
|
|||
${LIBMAMBA_INCLUDE_DIR}/mamba/solver/libsolv/solver.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/solver/libsolv/unsolvable.hpp
|
||||
# Artifacts validation
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/validation/tools.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/validation/errors.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/validation/keys.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/validation/update_framework.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/validation/repo_checker.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/validation/tools.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/validation/update_framework_v0_6.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/validation/update_framework_v1.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/validation/repo_checker.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/validation/update_framework.hpp
|
||||
# Downloaders and mirrors
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/download/downloader.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/download/mirror.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/download/mirror_map.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/download/mirror.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/download/request.hpp
|
||||
# Core API (low-level)
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/activation.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/channel_context.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/palette.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/common_types.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/context.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/download_progress_bar.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/env_lockfile.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/environments_manager.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/package_database_loader.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/error_handling.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/execution.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/fsutil.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/history.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/invoke.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/link.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/menuinst.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/mirror.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/output.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/package_cache.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/package_database_loader.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/package_fetcher.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/package_handling.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/package_paths.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/palette.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/pinning.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/prefix_data.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/progress_bar.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/pinning.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/query.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/repo_checker_store.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/run.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/shell_init.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/subdirdata.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/tasksync.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/thread_utils.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/transaction.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/timeref.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/transaction_context.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/util.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/fsutil.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/transaction.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/util_os.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/util_scope.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/util.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/virtual_packages.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/env_lockfile.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/tasksync.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/invoke.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/core/timeref.hpp
|
||||
# API (high-level)
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/api/c_api.h
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/api/channel_loader.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/api/clean.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/api/config.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/api/configuration.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/api/configuration_impl.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/api/configuration.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/api/constants.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/api/create.hpp
|
||||
${LIBMAMBA_INCLUDE_DIR}/mamba/api/info.hpp
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ add_executable(
|
|||
src/main.cpp
|
||||
src/msvc_catch_string_view.cpp
|
||||
src/pool_data.cpp
|
||||
src/pool_data.hpp
|
||||
src/test_pool.cpp
|
||||
src/test_queue.cpp
|
||||
src/test_repo.cpp
|
||||
|
|
|
|||
|
|
@ -1,182 +0,0 @@
|
|||
// Copyright (c) 2023, QuantStack and Mamba Contributors
|
||||
//
|
||||
// Distributed under the terms of the BSD 3-Clause License.
|
||||
//
|
||||
// The full license is in the file LICENSE, distributed with this software.
|
||||
|
||||
#ifndef MAMBA_CORE_MIRROR_HPP
|
||||
#define MAMBA_CORE_MIRROR_HPP
|
||||
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace mamba
|
||||
{
|
||||
class CURLHandle;
|
||||
|
||||
enum class Protocol
|
||||
{
|
||||
FILE,
|
||||
FTP,
|
||||
HTTP,
|
||||
OCI,
|
||||
};
|
||||
|
||||
// Statistics of mirror
|
||||
struct MirrorStats
|
||||
{
|
||||
// Maximum number of allowed parallel connections to this mirror. -1 means no
|
||||
// limit. Dynamically adjusted (decreased) if no fatal (temporary) error will
|
||||
// occur.
|
||||
// TODO To be set using `ctx.threads_params.download_threads`
|
||||
// Do we need it as an arg in Mirror constructor?
|
||||
// Anyway, it will need to be linked to (or replacing?) `m_max_parallel_downloads` in
|
||||
// CURLMultiHandle
|
||||
// Same for the other stats?
|
||||
long allowed_parallel_connections = -1;
|
||||
|
||||
// The maximum number of tried parallel connections to this mirror
|
||||
// (including unsuccessful).
|
||||
std::size_t max_tried_parallel_connections = 0;
|
||||
|
||||
// How many transfers from this mirror are currently in progress.
|
||||
std::size_t running_transfers = 0;
|
||||
|
||||
// How many transfers was finished successfully from the mirror.
|
||||
std::size_t successful_transfers = 0;
|
||||
|
||||
// How many transfers failed.
|
||||
std::size_t failed_transfers = 0;
|
||||
|
||||
// Maximum ranges supported in a single request. This will be automatically
|
||||
// adjusted when mirrors respond with 200 to a range request
|
||||
// zchunk case
|
||||
std::size_t max_ranges = 256;
|
||||
};
|
||||
|
||||
class Mirror
|
||||
{
|
||||
public:
|
||||
|
||||
Mirror(const std::string& url);
|
||||
|
||||
virtual ~Mirror();
|
||||
|
||||
Mirror(const Mirror&) = delete;
|
||||
Mirror& operator=(const Mirror&) = delete;
|
||||
Mirror(Mirror&&) = delete;
|
||||
Mirror& operator=(Mirror&&) = delete;
|
||||
|
||||
Protocol protocol() const;
|
||||
const std::string& url() const;
|
||||
std::chrono::system_clock::time_point next_retry() const;
|
||||
const MirrorStats& stats() const;
|
||||
|
||||
void set_max_ranges(std::size_t max_ranges);
|
||||
void set_allowed_parallel_connections(long allowed_parallel_connections);
|
||||
|
||||
bool need_wait_for_retry() const;
|
||||
bool has_running_transfers() const;
|
||||
|
||||
void increase_running_transfers();
|
||||
bool is_parallel_connections_limited_and_reached() const;
|
||||
void update_statistics(bool transfer_success);
|
||||
|
||||
virtual std::string format_url(const std::string& path) const;
|
||||
virtual std::string get_auth_header(const std::string& path) const;
|
||||
virtual bool needs_preparation(const std::string& path) const;
|
||||
virtual void prepare(const std::string& path, CURLHandle& handle);
|
||||
|
||||
protected:
|
||||
|
||||
Protocol m_protocol;
|
||||
|
||||
private:
|
||||
|
||||
const std::string m_url;
|
||||
|
||||
// TODO put these in a struct?
|
||||
// Retry & backoff params
|
||||
std::chrono::system_clock::time_point m_next_retry;
|
||||
std::chrono::system_clock::duration m_retry_wait_seconds;
|
||||
std::size_t m_retry_backoff_factor;
|
||||
// count number of retries (this is not the same as failed transfers, as multiple
|
||||
// transfers can be started at the same time, but should all be retried only once)
|
||||
std::size_t m_retry_counter;
|
||||
|
||||
MirrorStats m_stats;
|
||||
};
|
||||
|
||||
class HTTPMirror : public Mirror
|
||||
{
|
||||
public:
|
||||
|
||||
HTTPMirror(const std::string& url);
|
||||
~HTTPMirror();
|
||||
|
||||
bool authenticate(CURLHandle& handle, const std::string& user, const std::string& password);
|
||||
};
|
||||
|
||||
// Utility function
|
||||
// TODO leave it here or put it in a namespace, or move it to utils?
|
||||
std::pair<std::string, std::string> split_path_tag(const std::string& path);
|
||||
|
||||
class OCIMirror : public Mirror
|
||||
{
|
||||
public:
|
||||
|
||||
using proxy_map_type = std::map<std::string, std::string>;
|
||||
|
||||
OCIMirror(const std::string& host, const std::string& repo_prefix, const proxy_map_type& proxy_map);
|
||||
|
||||
OCIMirror(
|
||||
const std::string& host,
|
||||
const std::string& repo_prefix,
|
||||
const std::string& scope,
|
||||
const std::string& username,
|
||||
const std::string& password,
|
||||
const proxy_map_type& proxy_map
|
||||
);
|
||||
~OCIMirror();
|
||||
|
||||
|
||||
std::string get_repo(const std::string& repo) const;
|
||||
std::string get_auth_url(const std::string& repo, const std::string& scope) const;
|
||||
std::string get_manifest_url(const std::string& repo, const std::string& reference) const;
|
||||
std::string get_preupload_url(const std::string& repo) const;
|
||||
|
||||
bool need_auth() const;
|
||||
|
||||
std::string format_url(const std::string& path) const override;
|
||||
std::string get_auth_header(const std::string& path) const override;
|
||||
bool needs_preparation(const std::string& path) const override;
|
||||
void prepare(const std::string& path, CURLHandle& handle) override;
|
||||
|
||||
private:
|
||||
|
||||
struct AuthData
|
||||
{
|
||||
std::string sha256sum; // TODO what about other checksums types? i.e md5
|
||||
std::string token;
|
||||
};
|
||||
|
||||
std::map<std::string, std::unique_ptr<AuthData>> m_path_map;
|
||||
std::string m_repo_prefix;
|
||||
std::string m_scope;
|
||||
std::string m_username;
|
||||
std::string m_password;
|
||||
|
||||
// TODO do we really need this here? (only needs in `prepare` so far, add it as an arg
|
||||
// there?)
|
||||
proxy_map_type m_proxy_map;
|
||||
|
||||
AuthData* get_data(const std::string& path) const;
|
||||
};
|
||||
|
||||
} // namespace mamba
|
||||
|
||||
#endif // MAMBA_CORE_MIRROR_HPP
|
||||
|
|
@ -1,444 +0,0 @@
|
|||
// Copyright (c) 2023, QuantStack and Mamba Contributors
|
||||
//
|
||||
// Distributed under the terms of the BSD 3-Clause License.
|
||||
//
|
||||
// The full license is in the file LICENSE, distributed with this software.
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
#include "mamba/core/mirror.hpp"
|
||||
#include "mamba/core/output.hpp"
|
||||
#include "mamba/util/string.hpp"
|
||||
#include "mamba/util/url_manip.hpp"
|
||||
|
||||
#include "../download/curl.hpp"
|
||||
|
||||
namespace mamba
|
||||
{
|
||||
|
||||
/*************************
|
||||
* Mirror implementation *
|
||||
*************************/
|
||||
|
||||
Mirror::Mirror(const std::string& url)
|
||||
: m_protocol(Protocol::HTTP)
|
||||
, m_url(std::string(util::rstrip(url, '/')))
|
||||
, m_next_retry(std::chrono::system_clock::now())
|
||||
, m_retry_wait_seconds(std::chrono::milliseconds(200))
|
||||
, m_retry_backoff_factor(2)
|
||||
, m_retry_counter(0)
|
||||
{
|
||||
}
|
||||
|
||||
Mirror::~Mirror() = default;
|
||||
|
||||
Protocol Mirror::protocol() const
|
||||
{
|
||||
return m_protocol;
|
||||
}
|
||||
|
||||
const std::string& Mirror::url() const
|
||||
{
|
||||
return m_url;
|
||||
}
|
||||
|
||||
std::chrono::system_clock::time_point Mirror::next_retry() const
|
||||
{
|
||||
return m_next_retry;
|
||||
}
|
||||
|
||||
const MirrorStats& Mirror::stats() const
|
||||
{
|
||||
return m_stats;
|
||||
}
|
||||
|
||||
void Mirror::set_max_ranges(std::size_t max_ranges)
|
||||
{
|
||||
m_stats.max_ranges = max_ranges;
|
||||
}
|
||||
|
||||
void Mirror::set_allowed_parallel_connections(long allowed_parallel_connections)
|
||||
{
|
||||
m_stats.allowed_parallel_connections = allowed_parallel_connections;
|
||||
}
|
||||
|
||||
bool Mirror::need_wait_for_retry() const
|
||||
{
|
||||
return m_retry_counter != 0 && m_next_retry > std::chrono::system_clock::now();
|
||||
}
|
||||
|
||||
bool Mirror::has_running_transfers() const
|
||||
{
|
||||
return m_stats.running_transfers > 0;
|
||||
}
|
||||
|
||||
void Mirror::increase_running_transfers()
|
||||
{
|
||||
m_stats.running_transfers++;
|
||||
if (m_stats.max_tried_parallel_connections < m_stats.running_transfers)
|
||||
{
|
||||
m_stats.max_tried_parallel_connections = m_stats.running_transfers;
|
||||
}
|
||||
}
|
||||
|
||||
bool Mirror::is_parallel_connections_limited_and_reached() const
|
||||
{
|
||||
return m_stats.allowed_parallel_connections != -1
|
||||
&& m_stats.running_transfers >= std::size_t(m_stats.allowed_parallel_connections);
|
||||
}
|
||||
|
||||
void Mirror::update_statistics(bool transfer_success)
|
||||
{
|
||||
m_stats.running_transfers--;
|
||||
if (transfer_success)
|
||||
{
|
||||
m_stats.successful_transfers++;
|
||||
}
|
||||
else
|
||||
{
|
||||
m_stats.failed_transfers++;
|
||||
if (m_stats.failed_transfers == 1 || m_next_retry < std::chrono::system_clock::now())
|
||||
{
|
||||
m_retry_counter++;
|
||||
m_retry_wait_seconds = m_retry_wait_seconds * m_retry_backoff_factor;
|
||||
m_next_retry = std::chrono::system_clock::now() + m_retry_wait_seconds;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::string Mirror::format_url(const std::string& path) const
|
||||
{
|
||||
return util::url_concat(m_url, path);
|
||||
}
|
||||
|
||||
std::string Mirror::get_auth_header(const std::string&) const
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
bool Mirror::needs_preparation(const std::string&) const
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
void Mirror::prepare(const std::string&, CURLHandle&)
|
||||
{
|
||||
}
|
||||
|
||||
/*****************************
|
||||
* HTTPMirror implementation *
|
||||
*****************************/
|
||||
|
||||
HTTPMirror::HTTPMirror(const std::string& url)
|
||||
: Mirror(url)
|
||||
{
|
||||
}
|
||||
|
||||
HTTPMirror::~HTTPMirror() = default;
|
||||
|
||||
// TODO Maybe move this curl stuff somewhere else?
|
||||
// Or remove it if authentication won't be used
|
||||
bool
|
||||
HTTPMirror::authenticate(CURLHandle& handle, const std::string& user, const std::string& password)
|
||||
{
|
||||
if (!password.empty() && !user.empty())
|
||||
{
|
||||
handle.set_opt(CURLOPT_USERNAME, user);
|
||||
handle.set_opt(CURLOPT_PASSWORD, password);
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_WARNING << "Cannot authenticate: user or password empty";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Utility function
|
||||
std::pair<std::string, std::string> split_path_tag(const std::string& path)
|
||||
{
|
||||
// for OCI, if we have a filename like "xtensor-0.23.10-h2acdbc0_0.tar.bz2"
|
||||
// we want to split it to `xtensor:0.23.10-h2acdbc0-0`
|
||||
if (util::ends_with(path, ".json"))
|
||||
{
|
||||
return { path, "latest" };
|
||||
}
|
||||
|
||||
std::pair<std::string, std::string> result;
|
||||
auto parts = util::rsplit(path, "-", 2);
|
||||
|
||||
if (parts.size() < 2)
|
||||
{
|
||||
LOG_ERROR << "Could not split filename into enough parts";
|
||||
throw std::runtime_error("Could not split filename into enough parts");
|
||||
}
|
||||
|
||||
result.first = parts[0];
|
||||
|
||||
std::string tag;
|
||||
if (parts.size() > 2)
|
||||
{
|
||||
std::string last_part = parts[2].substr(0, parts[2].find_first_of("."));
|
||||
tag = fmt::format("{}-{}", parts[1], last_part);
|
||||
}
|
||||
else
|
||||
{
|
||||
tag = parts[1];
|
||||
}
|
||||
|
||||
util::replace_all(tag, "_", "-");
|
||||
result.second = tag;
|
||||
|
||||
LOG_INFO << "Splitting " << path << " to name: " << result.first << " tag: " << result.second;
|
||||
return result;
|
||||
}
|
||||
|
||||
/*****************************
|
||||
* OCIMirror implementation *
|
||||
*****************************/
|
||||
|
||||
// OCI Mirror:
|
||||
// When knowing the SHA256 we can directly get to the blob
|
||||
// When we do not know the SHA256 sum, we need to find the `latest` or some
|
||||
// other blob
|
||||
|
||||
// OCI upload process
|
||||
// 4 steps:
|
||||
// - first get auth token with push rights
|
||||
// - then
|
||||
|
||||
// This is what an OCI manifest (index) looks like:
|
||||
// {
|
||||
// "schemaVersion": 2,
|
||||
// "config": {
|
||||
// "mediaType": "application/vnd.unknown.config.v1+json",
|
||||
// "digest":
|
||||
// "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
// "size": 0
|
||||
// },
|
||||
// "layers": [
|
||||
// {
|
||||
// "mediaType": "application/vnd.unknown.layer.v1+txt",
|
||||
// "digest":
|
||||
// "sha256:c5be3ea75353851e1fcf3a298af3b6cfd2af3d7ff018ce52657b6dbd8f986aa4",
|
||||
// "size": 13,
|
||||
// "annotations": {
|
||||
// "org.opencontainers.image.title": "artifact.txt"
|
||||
// }
|
||||
// }
|
||||
// ]
|
||||
// }
|
||||
|
||||
// Used for pull/read (doesn't need auth)
|
||||
OCIMirror::OCIMirror(
|
||||
const std::string& host,
|
||||
const std::string& repo_prefix,
|
||||
const proxy_map_type& proxy_map
|
||||
)
|
||||
: Mirror(host)
|
||||
, m_repo_prefix(repo_prefix)
|
||||
, m_scope("pull")
|
||||
, m_proxy_map(proxy_map)
|
||||
{
|
||||
m_protocol = Protocol::OCI;
|
||||
}
|
||||
|
||||
OCIMirror::OCIMirror(
|
||||
const std::string& host,
|
||||
const std::string& repo_prefix,
|
||||
const std::string& scope,
|
||||
const std::string& username,
|
||||
const std::string& password,
|
||||
const proxy_map_type& proxy_map
|
||||
)
|
||||
: Mirror(host)
|
||||
, m_repo_prefix(repo_prefix)
|
||||
, m_scope(scope)
|
||||
, m_username(username)
|
||||
, m_password(password)
|
||||
, m_proxy_map(proxy_map)
|
||||
{
|
||||
m_protocol = Protocol::OCI;
|
||||
}
|
||||
|
||||
OCIMirror::~OCIMirror() = default;
|
||||
|
||||
std::string OCIMirror::get_repo(const std::string& repo) const
|
||||
{
|
||||
if (!m_repo_prefix.empty())
|
||||
{
|
||||
return fmt::format("{}/{}", m_repo_prefix, repo);
|
||||
}
|
||||
else
|
||||
{
|
||||
return repo;
|
||||
}
|
||||
}
|
||||
|
||||
std::string OCIMirror::get_auth_url(const std::string& repo, const std::string& scope) const
|
||||
{
|
||||
return fmt::format("{}/token?scope=repository:{}:{}", this->url(), get_repo(repo), scope);
|
||||
}
|
||||
|
||||
std::string OCIMirror::get_manifest_url(const std::string& repo, const std::string& reference) const
|
||||
{
|
||||
return fmt::format("{}/v2/{}/manifests/{}", this->url(), get_repo(repo), reference);
|
||||
}
|
||||
|
||||
std::string OCIMirror::get_preupload_url(const std::string& repo) const
|
||||
{
|
||||
return fmt::format("{}/v2/{}/blobs/uploads/", this->url(), get_repo(repo));
|
||||
}
|
||||
|
||||
bool OCIMirror::need_auth() const
|
||||
{
|
||||
return !m_username.empty() && !m_password.empty();
|
||||
}
|
||||
|
||||
std::string OCIMirror::format_url(const std::string& path) const
|
||||
{
|
||||
// TODO Maybe we would need a std::string checksum to pass as arg (to be empty in Mirror)
|
||||
// (in case the map data is not set yet)
|
||||
// to be checked in use/call of this method
|
||||
|
||||
auto* data = get_data(path);
|
||||
if (data)
|
||||
{
|
||||
auto [split_path, split_tag] = split_path_tag(path);
|
||||
// Should be this format:
|
||||
// https://ghcr.io/v2/wolfv/artifact/blobs/sha256:c5be3ea75353851e1fcf3a298af3b6cfd2af3d7ff018ce52657b6dbd8f986aa4
|
||||
return fmt::format(
|
||||
"{}/v2/{}/blobs/sha256:{}",
|
||||
this->url(),
|
||||
get_repo(split_path),
|
||||
data->sha256sum
|
||||
);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_ERROR << "Checksum corresponding to " << path << " is not available";
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
std::string OCIMirror::get_auth_header(const std::string& path) const
|
||||
{
|
||||
if (m_username.empty() && m_password.empty())
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
auto* data = get_data(path);
|
||||
if (data && !data->token.empty())
|
||||
{
|
||||
return fmt::format("Authorization: Bearer {}", data->token);
|
||||
}
|
||||
else
|
||||
{
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
bool OCIMirror::needs_preparation(const std::string& path) const
|
||||
{
|
||||
auto* data = get_data(path);
|
||||
if ((!data || (data && data->token.empty())) && need_auth())
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!data || (data && data->sha256sum.empty()))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
void OCIMirror::prepare(const std::string& path, CURLHandle& handle)
|
||||
{
|
||||
auto [split_path, split_tag] = split_path_tag(path);
|
||||
|
||||
auto* data = get_data(path);
|
||||
if (!data)
|
||||
{
|
||||
m_path_map[split_path].reset(new AuthData);
|
||||
data = m_path_map[split_path].get();
|
||||
}
|
||||
|
||||
if (data && data->token.empty() && need_auth())
|
||||
{
|
||||
std::string auth_url = get_auth_url(split_path, m_scope);
|
||||
|
||||
// TODO to be checked
|
||||
// Not sure this is necessary
|
||||
// Do we need to call curl config beforehand using the right args in
|
||||
// `configure_curl_handle`
|
||||
handle.set_url(auth_url, m_proxy_map);
|
||||
// IMPORTANT: Make sure the callbacks are set before calling this
|
||||
// i.e CURLOPT_HEADERFUNCTION, CURLOPT_HEADERDATA, CURLOPT_WRITEFUNCTION,
|
||||
// CURLOPT_WRITEDATA
|
||||
|
||||
if (!m_username.empty())
|
||||
{
|
||||
handle.set_opt(CURLOPT_USERNAME, m_username);
|
||||
}
|
||||
|
||||
if (!m_password.empty())
|
||||
{
|
||||
handle.set_opt(CURLOPT_PASSWORD, m_password);
|
||||
}
|
||||
|
||||
auto set_data_callback =
|
||||
[&data](bool success, const std::string& token, const std::string&) // Args got as
|
||||
// a response
|
||||
// to transfer
|
||||
{
|
||||
if (success && !token.empty())
|
||||
{
|
||||
data->token = token;
|
||||
}
|
||||
};
|
||||
|
||||
// TODO set here a callback to `set_data_callback` in CURLHandle or other relevant
|
||||
// class which performs the transfer finalization and sets data token
|
||||
// handle.set_end_callback(set_data_callback);
|
||||
}
|
||||
else
|
||||
{
|
||||
std::string manifest_url = get_manifest_url(split_path, split_tag);
|
||||
|
||||
handle.set_url(manifest_url, m_proxy_map)
|
||||
.add_header(get_auth_header(path))
|
||||
.add_header("Accept: application/vnd.oci.image.manifest.v1+json");
|
||||
|
||||
auto set_data_callback =
|
||||
[&data](bool success, const std::string&, const std::string& digest)
|
||||
{
|
||||
// digest is got from a json response like
|
||||
// j["layers"][0]["digest"]
|
||||
if (success && !digest.empty())
|
||||
{
|
||||
assert(util::starts_with(digest, "sha256:"));
|
||||
data->sha256sum = digest.substr(sizeof("sha256:") - 1);
|
||||
}
|
||||
};
|
||||
|
||||
// TODO set here a callback to `set_data_callback` in CURLHandle or other
|
||||
// relevant class which performs the transfer finalization and sets data sha256sum
|
||||
// handle.set_end_callback(set_data_callback);
|
||||
}
|
||||
}
|
||||
|
||||
OCIMirror::AuthData* OCIMirror::get_data(const std::string& path) const
|
||||
{
|
||||
auto [split_path, _] = split_path_tag(path);
|
||||
auto it = m_path_map.find(split_path);
|
||||
if (it != m_path_map.end())
|
||||
{
|
||||
return it->second.get();
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
} // namespace mamba
|
||||
|
|
@ -54,12 +54,12 @@ set(
|
|||
src/specs/test_regex_spec.cpp
|
||||
src/specs/test_repo_data.cpp
|
||||
src/specs/test_unresolved_channel.cpp
|
||||
src/specs/test_version.cpp
|
||||
src/specs/test_version_spec.cpp
|
||||
src/specs/test_version.cpp
|
||||
# Solver tests
|
||||
src/solver/test_problems_graph.cpp
|
||||
src/solver/test_request.cpp
|
||||
src/solver/test_solution.cpp
|
||||
src/solver/test_problems_graph.cpp
|
||||
# Solver libsolv implementation tests
|
||||
src/solver/libsolv/test_database.cpp
|
||||
src/solver/libsolv/test_solver.cpp
|
||||
|
|
@ -77,21 +77,21 @@ set(
|
|||
src/core/test_configuration.cpp
|
||||
src/core/test_cpp.cpp
|
||||
src/core/test_env_file_reading.cpp
|
||||
src/core/test_env_lockfile.cpp
|
||||
src/core/test_environments_manager.cpp
|
||||
src/core/test_execution.cpp
|
||||
src/core/test_filesystem.cpp
|
||||
src/core/test_history.cpp
|
||||
src/core/test_invoke.cpp
|
||||
src/core/test_lockfile.cpp
|
||||
src/core/test_pinning.cpp
|
||||
src/core/test_output.cpp
|
||||
src/core/test_pinning.cpp
|
||||
src/core/test_progress_bar.cpp
|
||||
src/core/test_shell_init.cpp
|
||||
src/core/test_thread_utils.cpp
|
||||
src/core/test_virtual_packages.cpp
|
||||
src/core/test_util.cpp
|
||||
src/core/test_env_lockfile.cpp
|
||||
src/core/test_execution.cpp
|
||||
src/core/test_invoke.cpp
|
||||
src/core/test_tasksync.cpp
|
||||
src/core/test_filesystem.cpp
|
||||
src/core/test_thread_utils.cpp
|
||||
src/core/test_util.cpp
|
||||
src/core/test_virtual_packages.cpp
|
||||
)
|
||||
|
||||
message(STATUS "Building libmamba C++ tests")
|
||||
|
|
|
|||
|
|
@ -17,8 +17,9 @@ set(
|
|||
longpath.manifest
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/activate.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/clean.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/config.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/common_options.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/completer.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/config.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/constructor.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/create.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/env.cpp
|
||||
|
|
@ -26,7 +27,6 @@ set(
|
|||
${CMAKE_CURRENT_SOURCE_DIR}/src/install.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/list.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/login.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/common_options.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/main.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/package.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/remove.cpp
|
||||
|
|
@ -41,6 +41,7 @@ set(
|
|||
set(
|
||||
MICROMAMBA_HEADERS
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/common_options.hpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/constructor.hpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/umamba.hpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/version.hpp
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in New Issue