folkslinux
/
mamba
mirror of https://github.com/mamba-org/mamba.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Simplify SubdirData (#3926) 

Co-authored-by: Julien Jerphanion <git@jjerphan.xyz>
This commit is contained in:
Antoine Prouvost 2025-05-12 19:21:19 +02:00 committed by GitHub
parent 1b3b9e1b25
commit 9ad464b126
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 463 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
libmamba/include/mamba/core/context.hpp
View File

@ -14,6 +14,7 @@
#include "mamba/core/common_types.hpp"
#include "mamba/core/palette.hpp"
#include "mamba/core/subdir_parameters.hpp"
#include "mamba/core/tasksync.hpp"
#include "mamba/download/mirror_map.hpp"
#include "mamba/download/parameters.hpp"
@ -174,6 +175,14 @@ namespace mamba
// micromamba only
bool shell_completion = true;
OutputParams output_params;
GraphicsParams graphics_params;
SrcParams src_params;
CommandParams command_params;
ThreadsParams threads_params;
PrefixParams prefix_params;
ValidationParams validation_params;
download::RemoteFetchParams remote_fetch_params = {
/* .ssl_verify */ { "" },
/* .ssl_no_revoke */ false,
@ -196,13 +205,15 @@ namespace mamba
};
}
OutputParams output_params;
GraphicsParams graphics_params;
SrcParams src_params;
CommandParams command_params;
ThreadsParams threads_params;
PrefixParams prefix_params;
ValidationParams validation_params;
SubdirParams subdir_params() const
{
return {
/* .local_repodata_ttl */ this->local_repodata_ttl,
/* .offline */ this->offline,
/* .use_index_cache */ this->use_index_cache,
/* .repodata_use_zst */ this->repodata_use_zst,
};
}
std::size_t lock_timeout = 0;
bool use_lockfiles = true;

23
libmamba/include/mamba/core/subdir_parameters.hpp Normal file
View File

@ -0,0 +1,23 @@
// Copyright (c) 2025, QuantStack and Mamba Contributors
//
// Distributed under the terms of the BSD 3-Clause License.
//
// The full license is in the file LICENSE, distributed with this software.
#ifndef MAMBA_CORE_SUBDIR_PARAMETERS_HPP
#define MAMBA_CORE_SUBDIR_PARAMETERS_HPP
#include <cstddef>
namespace mamba
{
struct SubdirParams
{
std::size_t local_repodata_ttl = 1;
bool offline = false;
bool use_index_cache = true;
bool repodata_use_zst = true;
};
}
#endif

273
libmamba/include/mamba/core/subdirdata.hpp
View File

@ -14,9 +14,14 @@
#include "mamba/core/error_handling.hpp"
#include "mamba/core/package_cache.hpp"
#include "mamba/core/subdir_parameters.hpp"
#include "mamba/core/util.hpp"
#include "mamba/download/downloader.hpp"
#include "mamba/download/parameters.hpp"
#include "mamba/fs/filesystem.hpp"
#include "mamba/specs/channel.hpp"
#include "mamba/specs/conda_url.hpp"
#include "mamba/specs/platform.hpp"
namespace mamba
{
@ -25,10 +30,14 @@ namespace mamba
class Channel;
}
class Context;
class ChannelContext;
class DownloadMonitor;
/**
* Handling of a subdirectory metadata.
*
* These metadata are used and stored to check if a subdirectory index is up to date,
* where it comes from, and what protocols are supported to fetch it.
*/
class SubdirMetadata
{
public:
@ -43,37 +52,45 @@ namespace mamba
using expected_subdir_metadata = tl::expected<SubdirMetadata, mamba_error>;
static expected_subdir_metadata read(const fs::u8path& file);
void write(const fs::u8path& file);
bool check_valid_metadata(const fs::u8path& file);
/** Read the metadata from a lightweight file containing only these metadata. */
static auto read_state_file( //
const fs::u8path& state_file,
const fs::u8path& repodata_file
) -> expected_subdir_metadata;
const std::string& url() const;
const std::string& etag() const;
const std::string& last_modified() const;
const std::string& cache_control() const;
/** Read the metadata from the ``repodata.json`` header. */
static auto read_from_repodata_json(const fs::u8path& json) -> expected_subdir_metadata;
bool has_zst() const;
/** Read from any of state file or ``repodata.json`` depending on extension. */
static auto read(const fs::u8path& file) -> expected_subdir_metadata;
void store_http_metadata(HttpMetadata data);
void store_file_metadata(const fs::u8path& file);
[[nodiscard]] auto is_valid_metadata(const fs::u8path& file) const -> bool;
[[nodiscard]] auto url() const -> const std::string&;
[[nodiscard]] auto etag() const -> const std::string&;
[[nodiscard]] auto last_modified() const -> const std::string&;
[[nodiscard]] auto cache_control() const -> const std::string&;
/** Check if zst is available and freshly checked. */
[[nodiscard]] auto has_up_to_date_zst() const -> bool;
void set_http_metadata(HttpMetadata data);
void set_zst(bool value);
void store_file_metadata(const fs::u8path& file);
/** Write the metadata to a lightweight file. */
void write_state_file(const fs::u8path& file);
friend void to_json(nlohmann::json& j, const SubdirMetadata& data);
friend void from_json(const nlohmann::json& j, SubdirMetadata& data);
private:
static expected_subdir_metadata
from_state_file(const fs::u8path& state_file, const fs::u8path& repodata_file);
static expected_subdir_metadata from_repodata_file(const fs::u8path& json);
#ifdef _WIN32
using time_type = std::chrono::system_clock::time_point;
#else
using time_type = fs::file_time_type;
#endif
HttpMetadata m_http;
time_type m_stored_mtime;
std::size_t m_stored_file_size;
struct CheckedAt
{
bool value;
@ -82,127 +99,149 @@ namespace mamba
bool has_expired() const;
};
HttpMetadata m_http;
std::optional<CheckedAt> m_has_zst;
time_type m_stored_mtime;
std::size_t m_stored_file_size;
friend void to_json(nlohmann::json& j, const CheckedAt& ca);
friend void from_json(const nlohmann::json& j, CheckedAt& ca);
friend void to_json(nlohmann::json& j, const SubdirMetadata& data);
friend void from_json(const nlohmann::json& j, SubdirMetadata& data);
};
/**
* Represents a channel subdirectory (i.e. a platform)
* packages index. Handles downloading of the index
* from the server and cache generation as well.
* Channel sub-directory (i.e. a platform) packages index.
*
* Handles downloading of the index from the server and cache generation.
* This only handles traditional ``repodata.json`` full indexes.
* This abstraction does not load the index in memory, with is done by the @ref Database.
*
* Upon creation, the caches are checked for a valid and up to date index.
* This can be inspected with @ref valid_cache_found.
* The created subdirs are typically used with @ref SubdirData::download_required_indexes
* which will download the missing, invalid, or outdated indexes as needed.
*/
class SubdirData
{
public:
static expected_t<SubdirData> create(
Context& ctx,
ChannelContext& channel_context,
const specs::Channel& channel,
const std::string& platform,
MultiPackageCache& caches,
const std::string& repodata_fn = "repodata.json"
);
~SubdirData() = default;
SubdirData(const SubdirData&) = delete;
SubdirData& operator=(const SubdirData&) = delete;
SubdirData(SubdirData&&) = default;
SubdirData& operator=(SubdirData&&) = default;
bool is_noarch() const;
bool is_loaded() const;
void clear_cache();
const std::string& name() const;
const std::string& channel_id() const;
const std::string& platform() const;
const SubdirMetadata& metadata() const;
expected_t<fs::u8path> valid_solv_cache() const;
fs::u8path writable_solv_cache() const;
expected_t<fs::u8path> valid_json_cache() const;
[[deprecated("since version 2.0 use ``valid_solv_cache`` or ``valid_json_cache`` instead")]]
expected_t<std::string> cache_path() const;
static expected_t<void> download_indexes(
/**
* Download the missing, invalid, or outdated indexes as needed in parallel.
*
* It first creates check requests to update some metadata, then download the indexes.
* The result can be inspected with the input subdirs methods, such as
* @ref valid_cache_found, @ref valid_json_cache_path etc.
*/
[[nodiscard]] static auto download_required_indexes(
std::vector<SubdirData>& subdirs,
const Context& context,
const SubdirParams& subdir_params,
const specs::AuthenticationDataBase& auth_info,
const download::mirror_map& mirrors,
const download::Options& download_options,
const download::RemoteFetchParams& remote_fetch_params,
download::Monitor* check_monitor = nullptr,
download::Monitor* download_monitor = nullptr
);
) -> expected_t<void>;
/** Check existing caches for a valid index validity and freshness. */
static auto create(
const SubdirParams& params,
ChannelContext& channel_context,
specs::Channel channel,
specs::DynamicPlatform platform,
MultiPackageCache& caches,
std::string repodata_filename = "repodata.json"
) -> expected_t<SubdirData>;
[[nodiscard]] auto is_noarch() const -> bool;
[[nodiscard]] auto is_local() const -> bool;
[[nodiscard]] auto channel() const -> const specs::Channel&;
[[nodiscard]] auto name() const -> std::string;
[[nodiscard]] auto channel_id() const -> const std::string&;
[[nodiscard]] auto platform() const -> const specs::DynamicPlatform&;
[[nodiscard]] auto metadata() const -> const SubdirMetadata&;
[[nodiscard]] auto repodata_url() const -> specs::CondaURL;
[[nodiscard]] auto caching_is_forbidden() const -> bool;
[[nodiscard]] auto valid_cache_found() const -> bool;
[[nodiscard]] auto valid_libsolv_cache_path() const -> expected_t<fs::u8path>;
[[nodiscard]] auto writable_libsolv_cache_path() const -> fs::u8path;
[[nodiscard]] auto valid_json_cache_path() const -> expected_t<fs::u8path>;
void clear_cache_files();
private:
static std::string get_name(const std::string& channel_id, const std::string& platform);
SubdirData(
Context& ctx,
ChannelContext& channel_context,
const specs::Channel& channel,
const std::string& platform,
MultiPackageCache& caches,
const std::string& repodata_fn = "repodata.json"
);
std::string repodata_url_path() const;
const std::string& repodata_full_url() const;
void
load(MultiPackageCache& caches, ChannelContext& channel_context, const specs::Channel& channel);
void load_cache(MultiPackageCache& caches);
void update_metadata_zst(ChannelContext& context, const specs::Channel& channel);
download::MultiRequest build_check_requests();
download::Request build_index_request();
expected_t<void> use_existing_cache();
expected_t<void> finalize_transfer(SubdirMetadata::HttpMetadata http_data);
void refresh_last_write_time(const fs::u8path& json_file, const fs::u8path& solv_file);
bool m_loaded = false;
bool m_forbid_cache = false;
bool m_json_cache_valid = false;
bool m_solv_cache_valid = false;
SubdirMetadata m_metadata;
specs::Channel m_channel;
fs::u8path m_valid_cache_path;
fs::u8path m_expired_cache_path;
fs::u8path m_writable_pkgs_dir;
std::string m_channel_id;
std::string m_platform;
std::string m_name;
std::string m_repodata_fn;
std::string m_json_fn;
std::string m_solv_fn;
bool m_is_noarch;
std::string m_full_url;
SubdirMetadata m_metadata;
specs::DynamicPlatform m_platform;
std::string m_repodata_filename;
std::string m_json_filename;
std::string m_solv_filename;
bool m_valid_cache_found = false;
bool m_json_cache_valid = false;
bool m_solv_cache_valid = false;
std::unique_ptr<TemporaryFile> m_temp_file;
const Context* p_context;
SubdirData(
const SubdirParams& params,
ChannelContext& channel_context,
specs::Channel channel,
std::string platform,
MultiPackageCache& caches,
std::string repodata_fn = "repodata.json"
);
[[nodiscard]] auto repodata_url_path() const -> std::string;
void load(
const MultiPackageCache& caches,
ChannelContext& channel_context,
const SubdirParams& params,
const specs::Channel& channel
);
void load_cache(const MultiPackageCache& caches, const SubdirParams& params);
void update_metadata_zst(
ChannelContext& context,
const SubdirParams& params,
const specs::Channel& channel
);
auto build_check_requests(const SubdirParams& params) -> download::MultiRequest;
auto build_index_request() -> download::Request;
auto use_existing_cache() -> expected_t<void>;
auto finalize_transfer(SubdirMetadata::HttpMetadata http_data) -> expected_t<void>;
void refresh_last_write_time(const fs::u8path& json_file, const fs::u8path& solv_file);
};
[[nodiscard]] std::string cache_name_from_url(std::string_view url);
/**
* Compute an id from a URL.
*
* This is intended to keep unique, filesystem-safe, cache entries in the cache directory.
* @see cache_filename_from_url
*/
[[nodiscard]] auto cache_name_from_url(std::string url) -> std::string;
// Contrary to conda original function, this one expects a full url
// (that is channel url + / + repodata_fn). It is not the
// responsibility of this function to decide whether it should
// concatenate base url and repodata depending on repodata value
// and old behavior support.
std::string cache_fn_url(const std::string& url);
std::string create_cache_dir(const fs::u8path& cache_path);
/**
* Compute a filename from a URL.
*
* This is intended to keep unique, filesystem-safe, cache entries in the cache directory.
* Contrary to conda original function, this one expects a full url (that is
* channel url + / + repodata_fn).
* It is not the responsibility of this function to decide whether it should concatenate base
* url and repodata depending on repodata value and old behavior support.
*/
[[nodiscard]] auto cache_filename_from_url(std::string url) -> std::string;
} // namespace mamba
/**
* Create cache directory with correct permissions
*
* @return The path to the directory created
*/
auto create_cache_dir(const fs::u8path& cache_path) -> std::string;
#endif // MAMBA_SUBDIRDATA_HPP
}
#endif

3
libmamba/include/mamba/specs/platform.hpp
View File

@ -78,6 +78,9 @@ namespace mamba::specs
[[nodiscard]] auto platform_is_win(KnownPlatform plat) -> bool;
[[nodiscard]] auto platform_is_win(DynamicPlatform plat) -> bool;
[[nodiscard]] auto platform_is_noarch(KnownPlatform plat) -> bool;
[[nodiscard]] auto platform_is_noarch(DynamicPlatform plat) -> bool;
/**
* Detect the platform on which mamba was built.
*/

26
libmamba/src/api/channel_loader.cpp
View File

@ -77,7 +77,7 @@ namespace mamba
}
auto sdires = SubdirData::create(
ctx,
ctx.subdir_params(),
channel_context,
channel,
platform,
@ -204,11 +204,27 @@ namespace mamba
{
SubdirDataMonitor check_monitor({ true, true });
SubdirDataMonitor index_monitor;
download_res = SubdirData::download_indexes(subdirs, ctx, &check_monitor, &index_monitor);
download_res = SubdirData::download_required_indexes(
subdirs,
ctx.subdir_params(),
ctx.authentication_info(),
ctx.mirrors,
ctx.download_options(),
ctx.remote_fetch_params,
&check_monitor,
&index_monitor
);
}
else
{
download_res = SubdirData::download_indexes(subdirs, ctx);
download_res = SubdirData::download_required_indexes(
subdirs,
ctx.subdir_params(),
ctx.authentication_info(),
ctx.mirrors,
ctx.download_options(),
ctx.remote_fetch_params
);
}
if (!download_res)
@ -235,7 +251,7 @@ namespace mamba
for (std::size_t i = 0; i < subdirs.size(); ++i)
{
auto& subdir = subdirs[i];
if (!subdir.is_loaded())
if (!subdir.valid_cache_found())
{
if (!ctx.offline && subdir.is_noarch())
{
@ -267,7 +283,7 @@ namespace mamba
{
LOG_WARNING << "Could not load repodata.json for " << subdir.name()
<< ". Deleting cache, and retrying.";
subdir.clear_cache();
subdir.clear_cache_files();
loading_failed = true;
}
}

15
libmamba/src/core/package_database_loader.cpp
View File

@ -72,7 +72,7 @@ namespace mamba
// Solv files are too slow on Windows.
if (!util::on_win)
{
auto maybe_repo = subdir.valid_solv_cache().and_then(
auto maybe_repo = subdir.valid_libsolv_cache_path().and_then(
[&](fs::u8path&& solv_file)
{
return database.add_repo_from_native_serialization(
@ -89,7 +89,7 @@ namespace mamba
}
}
return subdir.valid_json_cache()
return subdir.valid_json_cache_path()
.and_then(
[&](fs::u8path&& repodata_json)
{
@ -115,13 +115,18 @@ namespace mamba
if (!util::on_win)
{
database
.native_serialize_repo(repo, subdir.writable_solv_cache(), expected_cache_origin)
.native_serialize_repo(
repo,
subdir.writable_libsolv_cache_path(),
expected_cache_origin
)
.or_else(
[&](const auto& err)
{
LOG_WARNING << R"(Fail to write native serialization to file ")"
<< subdir.writable_solv_cache() << R"(" for repo ")"
<< subdir.name() << ": " << err.what();
<< subdir.writable_libsolv_cache_path()
<< R"(" for repo ")" << subdir.name() << ": "
<< err.what();
;
}
);

304
libmamba/src/core/subdirdata.cpp
View File

@ -8,7 +8,6 @@
#include <stdexcept>
#include "mamba/core/channel_context.hpp"
#include "mamba/core/context.hpp"
#include "mamba/core/output.hpp"
#include "mamba/core/package_cache.hpp"
#include "mamba/core/subdirdata.hpp"
@ -115,10 +114,6 @@ namespace mamba
}
}
/*******************
* MSubdirMetadata *
*******************/
void to_json(nlohmann::json& j, const SubdirMetadata::CheckedAt& ca)
{
j["value"] = ca.value;
@ -168,22 +163,22 @@ namespace mamba
state_file.replace_extension(".state.json");
if (fs::is_regular_file(state_file))
{
return from_state_file(state_file, file);
return read_state_file(state_file, file);
}
else
{
return from_repodata_file(file);
return read_from_repodata_json(file);
}
}
void SubdirMetadata::write(const fs::u8path& file)
void SubdirMetadata::write_state_file(const fs::u8path& file)
{
nlohmann::json j = *this;
std::ofstream out = open_ofstream(file);
out << j.dump(4);
}
bool SubdirMetadata::check_valid_metadata(const fs::u8path& file)
bool SubdirMetadata::is_valid_metadata(const fs::u8path& file) const
{
if (const auto new_size = fs::file_size(file); new_size != m_stored_file_size)
{
@ -223,12 +218,12 @@ namespace mamba
return m_http.cache_control;
}
bool SubdirMetadata::has_zst() const
bool SubdirMetadata::has_up_to_date_zst() const
{
return m_has_zst.has_value() && m_has_zst.value().value && !m_has_zst.value().has_expired();
}
void SubdirMetadata::store_http_metadata(HttpMetadata data)
void SubdirMetadata::set_http_metadata(HttpMetadata data)
{
m_http = std::move(data);
}
@ -250,7 +245,7 @@ namespace mamba
}
auto
SubdirMetadata::from_state_file(const fs::u8path& state_file, const fs::u8path& repodata_file)
SubdirMetadata::read_state_file(const fs::u8path& state_file, const fs::u8path& repodata_file)
-> expected_subdir_metadata
{
std::ifstream infile = open_ifstream(state_file);
@ -275,7 +270,7 @@ namespace mamba
);
}
if (!m.check_valid_metadata(repodata_file))
if (!m.is_valid_metadata(repodata_file))
{
LOG_WARNING << "Cache file " << repodata_file << " was modified by another program";
return make_unexpected(
@ -286,7 +281,7 @@ namespace mamba
return m;
}
auto SubdirMetadata::from_repodata_file(const fs::u8path& repodata_file)
auto SubdirMetadata::read_from_repodata_json(const fs::u8path& repodata_file)
-> expected_subdir_metadata
{
const std::string json = [](const fs::u8path& file) -> std::string
@ -394,20 +389,43 @@ namespace mamba
fs::copy(new_file, old_file);
return old_file;
}
[[nodiscard]] auto get_name(std::string_view channel_id, std::string_view platform)
-> std::string
{
return util::url_concat(channel_id, "/", platform);
}
}
expected_t<SubdirData> SubdirData::create(
Context& ctx,
const SubdirParams& params,
ChannelContext& channel_context,
const specs::Channel& channel,
const std::string& platform,
specs::Channel channel,
specs::DynamicPlatform platform,
MultiPackageCache& caches,
const std::string& repodata_fn
std::string repodata_filename
)
{
if (channel.is_package())
{
return make_unexpected(
"Channel pointing to a single package artifacts do not have an index.",
mamba_error_code::incorrect_usage
);
}
auto name = get_name(channel.id(), platform);
try
{
return SubdirData(ctx, channel_context, channel, platform, caches, repodata_fn);
return SubdirData(
params,
channel_context,
std::move(channel),
std::move(platform),
caches,
std::move(repodata_filename)
);
}
catch (std::exception& e)
{
@ -416,8 +434,7 @@ namespace mamba
catch (...)
{
return make_unexpected(
"Unknown error when trying to load subdir data "
+ SubdirData::get_name(channel.id(), platform),
"Unknown error when trying to load subdir data " + name,
mamba_error_code::unknown
);
}
@ -425,37 +442,53 @@ namespace mamba
bool SubdirData::is_noarch() const
{
return m_is_noarch;
return specs::platform_is_noarch(m_platform);
}
bool SubdirData::is_loaded() const
auto SubdirData::is_local() const -> bool
{
return m_loaded;
return (channel().mirror_urls().size() == 1u) && (channel().url().scheme() == "file");
}
void SubdirData::clear_cache()
auto SubdirData::channel() const -> const specs::Channel&
{
if (fs::is_regular_file(m_json_fn))
return m_channel;
}
auto SubdirData::caching_is_forbidden() const -> bool
{
// The only condition yet
return is_local();
}
bool SubdirData::valid_cache_found() const
{
return m_valid_cache_found;
}
void SubdirData::clear_cache_files()
{
if (fs::is_regular_file(m_json_filename))
{
fs::remove(m_json_fn);
fs::remove(m_json_filename);
}
if (fs::is_regular_file(m_solv_fn))
if (fs::is_regular_file(m_solv_filename))
{
fs::remove(m_solv_fn);
fs::remove(m_solv_filename);
}
}
const std::string& SubdirData::name() const
std::string SubdirData::name() const
{
return m_name;
return get_name(channel_id(), m_platform);
}
const std::string& SubdirData::channel_id() const
{
return m_channel_id;
return m_channel.id();
}
const std::string& SubdirData::platform() const
const specs::DynamicPlatform& SubdirData::platform() const
{
return m_platform;
}
@ -465,46 +498,36 @@ namespace mamba
return m_metadata;
}
expected_t<fs::u8path> SubdirData::valid_solv_cache() const
expected_t<fs::u8path> SubdirData::valid_libsolv_cache_path() const
{
if (m_json_cache_valid && m_solv_cache_valid)
{
return (get_cache_dir(m_valid_cache_path) / m_solv_fn).string();
return (get_cache_dir(m_valid_cache_path) / m_solv_filename).string();
}
return make_unexpected("Cache not loaded", mamba_error_code::cache_not_loaded);
}
fs::u8path SubdirData::writable_solv_cache() const
fs::u8path SubdirData::writable_libsolv_cache_path() const
{
return m_writable_pkgs_dir / "cache" / m_solv_fn;
return m_writable_pkgs_dir / "cache" / m_solv_filename;
}
expected_t<fs::u8path> SubdirData::valid_json_cache() const
expected_t<fs::u8path> SubdirData::valid_json_cache_path() const
{
if (m_json_cache_valid)
{
return (get_cache_dir(m_valid_cache_path) / m_json_fn).string();
return (get_cache_dir(m_valid_cache_path) / m_json_filename).string();
}
return make_unexpected("Cache not loaded", mamba_error_code::cache_not_loaded);
}
expected_t<std::string> SubdirData::cache_path() const
{
// TODO invalidate solv cache on version updates!!
if (m_json_cache_valid && m_solv_cache_valid)
{
return (get_cache_dir(m_valid_cache_path) / m_solv_fn).string();
}
else if (m_json_cache_valid)
{
return (get_cache_dir(m_valid_cache_path) / m_json_fn).string();
}
return make_unexpected("Cache not loaded", mamba_error_code::cache_not_loaded);
}
expected_t<void> SubdirData::download_indexes(
expected_t<void> SubdirData::download_required_indexes(
std::vector<SubdirData>& subdirs,
const Context& context,
const SubdirParams& subdir_params,
const specs::AuthenticationDataBase& auth_info,
const download::mirror_map& mirrors,
const download::Options& download_options,
const download::RemoteFetchParams& remote_fetch_params,
download::Monitor* check_monitor,
download::Monitor* download_monitor
)
@ -512,18 +535,18 @@ namespace mamba
download::MultiRequest check_requests;
for (auto& subdir : subdirs)
{
if (!subdir.is_loaded())
if (!subdir.valid_cache_found())
{
download::MultiRequest check_list = subdir.build_check_requests();
download::MultiRequest check_list = subdir.build_check_requests(subdir_params);
std::move(check_list.begin(), check_list.end(), std::back_inserter(check_requests));
}
}
download::download(
std::move(check_requests),
context.mirrors,
context.remote_fetch_params,
context.authentication_info(),
context.download_options(),
mirrors,
remote_fetch_params,
auth_info,
download_options,
check_monitor
);
@ -533,12 +556,12 @@ namespace mamba
}
// TODO load local channels even when offline if (!ctx.offline)
if (!context.offline)
if (!subdir_params.offline)
{
download::MultiRequest index_requests;
for (auto& subdir : subdirs)
{
if (!subdir.is_loaded())
if (!subdir.valid_cache_found())
{
index_requests.push_back(subdir.build_index_request());
}
@ -548,10 +571,10 @@ namespace mamba
{
download::download(
std::move(index_requests),
context.mirrors,
context.remote_fetch_params,
context.authentication_info(),
context.download_options(),
mirrors,
remote_fetch_params,
auth_info,
download_options,
download_monitor
);
}
@ -564,57 +587,50 @@ namespace mamba
return expected_t<void>();
}
std::string SubdirData::get_name(const std::string& channel_id, const std::string& platform)
{
return util::url_concat(channel_id, "/", platform);
}
SubdirData::SubdirData(
Context& ctx,
const SubdirParams& params,
ChannelContext& channel_context,
const specs::Channel& channel,
const std::string& platform,
specs::Channel channel,
std::string platform,
MultiPackageCache& caches,
const std::string& repodata_fn
std::string repodata_filename
)
: m_valid_cache_path("")
: m_channel(std::move(channel))
, m_valid_cache_path("")
, m_expired_cache_path("")
, m_writable_pkgs_dir(caches.first_writable_path())
, m_channel_id(channel.id())
, m_platform(platform)
, m_name(get_name(m_channel_id, m_platform))
, m_repodata_fn(repodata_fn)
, m_json_fn(cache_fn_url(name()))
, m_solv_fn(m_json_fn.substr(0, m_json_fn.size() - 4) + "solv")
, m_is_noarch(platform == "noarch")
, p_context(&(ctx))
, m_platform(std::move(platform))
, m_repodata_filename(std::move(repodata_filename))
, m_json_filename(cache_filename_from_url(name()))
, m_solv_filename(m_json_filename.substr(0, m_json_filename.size() - 4) + "solv")
{
m_full_url = util::url_concat(channel.url().str(), "/", repodata_url_path());
assert(!channel.is_package());
m_forbid_cache = (channel.mirror_urls().size() == 1u)
&& util::starts_with(channel.url().str(), "file://");
load(caches, channel_context, channel);
assert(!this->channel().is_package());
load(caches, channel_context, params, this->channel());
}
std::string SubdirData::repodata_url_path() const
{
return util::concat(m_platform, "/", m_repodata_fn);
return util::url_concat(m_platform, "/", m_repodata_filename);
}
const std::string& SubdirData::repodata_full_url() const
specs::CondaURL SubdirData::repodata_url() const
{
return m_full_url;
return channel().platform_url(m_platform) / m_repodata_filename;
}
void
SubdirData::load(MultiPackageCache& caches, ChannelContext& channel_context, const specs::Channel& channel)
void SubdirData::load(
const MultiPackageCache& caches,
ChannelContext& channel_context,
const SubdirParams& params,
const specs::Channel& channel
)
{
if (!m_forbid_cache)
if (!caching_is_forbidden())
{
load_cache(caches);
load_cache(caches, params);
}
if (m_loaded)
if (m_valid_cache_found)
{
Console::stream() << fmt::format("{:<50} {:>20}", name(), std::string("Using cache"));
}
@ -626,22 +642,21 @@ namespace mamba
LOG_INFO << "Expired cache (or invalid mod/etag headers) found at '"
<< m_expired_cache_path.string() << "'";
}
update_metadata_zst(channel_context, channel);
update_metadata_zst(channel_context, params, channel);
}
}
void SubdirData::load_cache(MultiPackageCache& caches)
void SubdirData::load_cache(const MultiPackageCache& caches, const SubdirParams& params)
{
LOG_INFO << "Searching index cache file for repo '" << name() << "'";
file_time_point now = fs::file_time_type::clock::now();
const Context& context = *p_context;
const auto cache_paths = without_duplicates(caches.paths());
for (const fs::u8path& cache_path : cache_paths)
{
// TODO: rewrite this with pipe chains of ranges
fs::u8path json_file = cache_path / "cache" / m_json_fn;
fs::u8path json_file = cache_path / "cache" / m_json_filename;
if (!fs::is_regular_file(json_file))
{
continue;
@ -664,26 +679,26 @@ namespace mamba
const int max_age = get_max_age(
m_metadata.cache_control(),
static_cast<int>(context.local_repodata_ttl)
static_cast<int>(params.local_repodata_ttl)
);
const auto cache_age_seconds = std::chrono::duration_cast<std::chrono::seconds>(cache_age)
.count();
if ((max_age > cache_age_seconds || context.offline || context.use_index_cache))
if ((max_age > cache_age_seconds || params.offline || params.use_index_cache))
{
// valid json cache found
if (!m_loaded)
if (!m_valid_cache_found)
{
LOG_DEBUG << "Using JSON cache";
LOG_TRACE << "Cache age: " << cache_age_seconds << "/" << max_age << "s";
m_valid_cache_path = cache_path;
m_json_cache_valid = true;
m_loaded = true;
m_valid_cache_found = true;
}
// check libsolv cache
fs::u8path solv_file = cache_path / "cache" / m_solv_fn;
fs::u8path solv_file = cache_path / "cache" / m_solv_filename;
file_duration solv_age = get_cache_age(solv_file, now);
if (is_valid(solv_age) && solv_age <= cache_age)
@ -710,26 +725,28 @@ namespace mamba
}
}
void
SubdirData::update_metadata_zst(ChannelContext& channel_context, const specs::Channel& channel)
void SubdirData::update_metadata_zst(
ChannelContext& channel_context,
const SubdirParams& params,
const specs::Channel& channel
)
{
const Context& context = *p_context;
if (!context.offline || m_forbid_cache)
if (!params.offline || caching_is_forbidden())
{
m_metadata.set_zst(m_metadata.has_zst() || channel_context.has_zst(channel));
m_metadata.set_zst(m_metadata.has_up_to_date_zst() || channel_context.has_zst(channel));
}
}
download::MultiRequest SubdirData::build_check_requests()
download::MultiRequest SubdirData::build_check_requests(const SubdirParams& params)
{
download::MultiRequest request;
if ((!p_context->offline || m_forbid_cache) && p_context->repodata_use_zst
&& !m_metadata.has_zst())
if ((!params.offline || caching_is_forbidden()) && params.repodata_use_zst
&& !m_metadata.has_up_to_date_zst())
{
request.push_back(download::Request(
name() + " (check zst)",
download::MirrorName(m_channel_id),
download::MirrorName(channel_id()),
repodata_url_path() + ".zst",
"",
/* lhead_only = */ true,
@ -767,15 +784,15 @@ namespace mamba
auto lock = LockFile(writable_cache_dir);
m_temp_file = std::make_unique<TemporaryFile>("mambaf", "", writable_cache_dir);
bool use_zst = m_metadata.has_zst();
bool use_zst = m_metadata.has_up_to_date_zst();
download::Request request(
name(),
download::MirrorName(m_channel_id),
download::MirrorName(channel_id()),
repodata_url_path() + (use_zst ? ".zst" : ""),
m_temp_file->path().string(),
/*head_only*/ false,
/*ignore_failure*/ !m_is_noarch
/*ignore_failure*/ !is_noarch()
);
request.etag = m_metadata.etag();
request.last_modified = m_metadata.last_modified();
@ -788,10 +805,12 @@ namespace mamba
}
else
{
return finalize_transfer(SubdirMetadata::HttpMetadata{ repodata_full_url(),
success.etag,
success.last_modified,
success.cache_control });
return finalize_transfer(SubdirMetadata::HttpMetadata{
repodata_url().str(),
success.etag,
success.last_modified,
success.cache_control,
});
}
};
@ -820,8 +839,8 @@ namespace mamba
{
LOG_INFO << "Cache is still valid";
fs::u8path json_file = m_expired_cache_path / "cache" / m_json_fn;
fs::u8path solv_file = m_expired_cache_path / "cache" / m_solv_fn;
fs::u8path json_file = m_expired_cache_path / "cache" / m_json_filename;
fs::u8path solv_file = m_expired_cache_path / "cache" / m_solv_filename;
if (path::is_writable(json_file)
&& (!fs::is_regular_file(solv_file) || path::is_writable(solv_file)))
@ -845,12 +864,12 @@ namespace mamba
fs::u8path writable_cache_dir = get_cache_dir(m_writable_pkgs_dir);
auto lock = LockFile(writable_cache_dir);
fs::u8path copied_json_file = writable_cache_dir / m_json_fn;
fs::u8path copied_json_file = writable_cache_dir / m_json_filename;
json_file = replace_file(copied_json_file, json_file);
if (fs::is_regular_file(solv_file))
{
auto copied_solv_file = writable_cache_dir / m_solv_fn;
auto copied_solv_file = writable_cache_dir / m_solv_filename;
solv_file = replace_file(copied_solv_file, solv_file);
}
@ -860,7 +879,7 @@ namespace mamba
refresh_last_write_time(json_file, solv_file);
m_temp_file.reset();
m_loaded = true;
m_valid_cache_found = true;
return expected_t<void>();
}
@ -877,10 +896,10 @@ namespace mamba
LOG_DEBUG << "Finalized transfer of '" << http_data.url << "'";
m_metadata.store_http_metadata(std::move(http_data));
m_metadata.set_http_metadata(std::move(http_data));
fs::u8path writable_cache_dir = get_cache_dir(m_writable_pkgs_dir);
fs::u8path json_file = writable_cache_dir / m_json_fn;
fs::u8path json_file = writable_cache_dir / m_json_filename;
auto lock = LockFile(writable_cache_dir);
fs::u8path state_file = json_file;
@ -900,12 +919,12 @@ namespace mamba
}
m_metadata.store_file_metadata(json_file);
m_metadata.write(state_file);
m_metadata.write_state_file(state_file);
m_temp_file.reset();
m_valid_cache_path = m_writable_pkgs_dir;
m_json_cache_valid = true;
m_loaded = true;
m_valid_cache_found = true;
return expected_t<void>();
}
@ -934,29 +953,28 @@ namespace mamba
state_file.replace_extension(".state.json");
auto lock = LockFile(state_file);
m_metadata.store_file_metadata(json_file);
m_metadata.write(state_file);
m_metadata.write_state_file(state_file);
}
std::string cache_name_from_url(std::string_view url)
std::string cache_name_from_url(std::string url)
{
auto u = std::string(url);
if (u.empty() || (u.back() != '/' && !util::ends_with(u, ".json")))
if (url.empty() || (url.back() != '/' && !util::ends_with(url, ".json")))
{
u += '/';
url += '/';
}
// mimicking conda's behavior by special handling repodata.json
// todo support .zst
if (util::ends_with(u, "/repodata.json"))
if (util::ends_with(url, "/repodata.json"))
{
u = u.substr(0, u.size() - 13);
url = url.substr(0, url.size() - 13);
}
return util::Md5Hasher().str_hex_str(u).substr(0, 8u);
return util::Md5Hasher().str_hex_str(url).substr(0, 8u);
}
std::string cache_fn_url(const std::string& url)
std::string cache_filename_from_url(std::string url)
{
return cache_name_from_url(url) + ".json";
return cache_name_from_url(std::move(url)) + ".json";
}
std::string create_cache_dir(const fs::u8path& cache_path)
@ -993,4 +1011,4 @@ namespace mamba
return cache_dir.string();
}
} // namespace mamba
}

14
libmamba/src/specs/platform.cpp
View File

@ -68,7 +68,19 @@ namespace mamba::specs
auto platform_is_win(DynamicPlatform plat) -> bool
{
static constexpr auto repr = std::string_view("win");
return (plat.size() >= repr.size()) && util::starts_with(util::to_lower(plat), repr);
return (plat.size() >= repr.size())
&& util::starts_with(util::to_lower(std::move(plat)), repr);
}
auto platform_is_noarch(KnownPlatform plat) -> bool
{
return plat == KnownPlatform::noarch;
}
auto platform_is_noarch(DynamicPlatform plat) -> bool
{
static constexpr auto repr = std::string_view("noarch");
return util::starts_with(util::to_lower(std::move(plat)), repr);
}
/**

2
libmamba/tests/src/core/test_cpp.cpp
View File

@ -425,7 +425,7 @@ namespace mamba
REQUIRE(j.etag() == "something else");
REQUIRE(j.last_modified() == "Fri, 11 Feb 2022 13:52:44 GMT");
REQUIRE(j.url() == "https://conda.anaconda.org/conda-forge/noarch/repodata.json.zst");
REQUIRE(j.has_zst() == false);
REQUIRE(j.has_up_to_date_zst() == false);
}
}
} // namespace mamba

18
libmamba/tests/src/solver/test_problems_graph.cpp
View File

@ -342,14 +342,28 @@ namespace
create_mirrors(ctx, chan);
for (const auto& platform : chan.platforms())
{
auto sub_dir = SubdirData::create(ctx, channel_context, chan, platform, cache)
auto sub_dir = SubdirData::create(
ctx.subdir_params(),
channel_context,
chan,
platform,
cache
)
.value();
sub_dirs.push_back(std::move(sub_dir));
}
}
}
SubdirData::download_indexes(sub_dirs, mambatests::context());
const auto result = SubdirData::download_required_indexes(
sub_dirs,
mambatests::context().subdir_params(),
mambatests::context().authentication_info(),
mambatests::context().mirrors,
mambatests::context().download_options(),
mambatests::context().remote_fetch_params
);
REQUIRE(result.has_value());
for (auto& sub_dir : sub_dirs)
{

49
libmambapy/src/libmambapy/bindings/legacy.cpp
View File

@ -162,7 +162,7 @@ namespace mambapy
{
using namespace mamba;
m_subdirs.push_back(extract(
SubdirData::create(ctx, channel_context, channel, platform, caches, repodata_fn)
SubdirData::create(ctx.subdir_params(), channel_context, channel, platform, caches, repodata_fn)
));
m_entries.push_back({ nullptr, platform, &channel, url });
for (size_t i = 0; i < m_subdirs.size(); ++i)
@ -181,11 +181,27 @@ namespace mambapy
{
SubdirDataMonitor check_monitor({ true, true });
SubdirDataMonitor index_monitor;
download_res = SubdirData::download_indexes(m_subdirs, ctx, &check_monitor, &index_monitor);
download_res = SubdirData::download_required_indexes(
m_subdirs,
ctx.subdir_params(),
ctx.authentication_info(),
ctx.mirrors,
ctx.download_options(),
ctx.remote_fetch_params,
&check_monitor,
&index_monitor
);
}
else
{
download_res = SubdirData::download_indexes(m_subdirs, ctx);
download_res = SubdirData::download_required_indexes(
m_subdirs,
ctx.subdir_params(),
ctx.authentication_info(),
ctx.mirrors,
ctx.download_options(),
ctx.remote_fetch_params
);
}
return download_res.has_value();
}
@ -522,13 +538,13 @@ bind_submodule_impl(pybind11::module_ m)
py::arg("context"),
py::arg("db")
)
.def("loaded", &SubdirData::is_loaded)
.def("loaded", &SubdirData::valid_cache_found)
.def(
"valid_solv_cache",
// TODO make a proper well tested type caster for expected types.
[](const SubdirData& self) -> std::optional<fs::u8path>
{
if (auto f = self.valid_solv_cache())
if (auto f = self.valid_libsolv_cache_path())
{
return { *std::move(f) };
}
@ -539,7 +555,7 @@ bind_submodule_impl(pybind11::module_ m)
"valid_json_cache",
[](const SubdirData& self) -> std::optional<fs::u8path>
{
if (auto f = self.valid_json_cache())
if (auto f = self.valid_json_cache_path())
{
return { *std::move(f) };
}
@ -554,7 +570,15 @@ bind_submodule_impl(pybind11::module_ m)
"Use `SubdirData.valid_solv_path` or `SubdirData.valid_json` path instead",
"2.0"
);
return extract(self.cache_path());
if (auto solv_path = self.valid_libsolv_cache_path())
{
return solv_path->string();
}
else if (auto json_path = self.valid_json_cache_path())
{
return json_path->string();
}
throw mamba_error("Cache not loaded", mamba_error_code::cache_not_loaded);
}
);
@ -602,7 +626,16 @@ bind_submodule_impl(pybind11::module_ m)
py::keep_alive<0, 1>()
);
m.def("cache_fn_url", &cache_fn_url);
m.def("cache_name_from_url", &cache_name_from_url);
m.def(
"cache_fn_url",
[](std::string url)
{
deprecated("This function was renamed `cache_filename_from_url`.", "2.3.0");
return cache_filename_from_url(std::move(url));
}
);
m.def("cache_filename_from_url", &cache_filename_from_url);
m.def("create_cache_dir", &create_cache_dir);
py::enum_<ChannelPriority>(m, "ChannelPriority")