folkslinux
/
mamba
mirror of https://github.com/mamba-org/mamba.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add tests for etag/mod/etc reading in cache

This commit is contained in:
Wolf Vollprecht 2022-02-11 18:11:13 +01:00
parent 96d25104d6
commit 7e4c03245b
10 changed files with 207 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.pre-commit-config.yaml
View File

@ -1,3 +1,4 @@
exclude: libmamba/tests/repodata_json_cache*
repos:
- repo: https://github.com/psf/black
rev: 21.12b0

7
libmamba/include/mamba/core/subdirdata.hpp
View File

@ -29,6 +29,12 @@ namespace decompress
namespace mamba
{
namespace detail
{
// read the header that contains json like {"_mod": "...", ...}
nlohmann::json read_mod_and_etag(const fs::path& file);
}
/**
* Represents a channel subdirectory (i.e. a platform)
* packages index. Handles downloading of the index
@ -72,7 +78,6 @@ namespace mamba
bool decompress();
void create_target(nlohmann::json& mod_etag);
std::size_t get_cache_control_max_age(const std::string& val);
nlohmann::json read_mod_and_etag(const fs::path&);
std::unique_ptr<DownloadTarget> m_target;

153
libmamba/src/core/subdirdata.cpp
View File

@ -63,6 +63,102 @@ namespace decompress
namespace mamba
{
namespace detail
{
nlohmann::json read_mod_and_etag(const fs::path& file)
{
// parse json at the beginning of the stream such as
// {"_url": "https://conda.anaconda.org/conda-forge/linux-64",
// "_etag": "W/\"6092e6a2b6cec6ea5aade4e177c3edda-8\"",
// "_mod": "Sat, 04 Apr 2020 03:29:49 GMT",
// "_cache_control": "public, max-age=1200"
auto extract_subjson = [](std::ifstream& s) -> std::string {
char next;
std::string result;
bool escaped = false;
int i = 0, N = 4;
std::size_t idx = 0;
std::size_t prev_keystart;
bool in_key = false;
std::string key = "";
while (s.get(next))
{
idx++;
if (next == '"')
{
if (!escaped)
{
if ((i / 2) % 2 == 0)
{
in_key = !in_key;
if (in_key)
{
prev_keystart = idx + 1;
}
else
{
if (key != "_mod" && key != "_etag" && key != "_cache_control" && key != "_url")
{
// bail out
auto last_comma = result.find_last_of(",", prev_keystart - 2);
if (last_comma != std::string::npos && last_comma > 0)
{
result = result.substr(0, last_comma);
result.push_back('}');
return result;
}
else
{
return std::string();
}
}
key.clear();
}
}
i++;
}
// 4 keys == 4 ticks
if (i == 4 * N)
{
result += "\"}";
return result;
}
}
if (in_key && next != '"')
{
key.push_back(next);
}
escaped = (!escaped && next == '\\');
result.push_back(next);
}
return std::string();
};
std::ifstream in_file = open_ifstream(file);
auto json = extract_subjson(in_file);
nlohmann::json result;
try
{
result = nlohmann::json::parse(json);
return result;
}
catch (...)
{
LOG_WARNING << "Could not parse mod/etag header";
return nlohmann::json();
}
}
}
MSubdirData::MSubdirData(const std::string& name,
const std::string& repodata_url,
const std::string& repodata_fn,
@ -132,7 +228,7 @@ namespace mamba
if (cache_age != fs::file_time_type::duration::max() && !forbid_cache())
{
LOG_INFO << "Found cache at '" << json_file.string() << "'";
m_mod_etag = read_mod_and_etag(json_file);
m_mod_etag = detail::read_mod_and_etag(json_file);
if (m_mod_etag.size() != 0)
{
@ -478,61 +574,6 @@ namespace mamba
return std::stoi(max_age_match[1]);
}
nlohmann::json MSubdirData::read_mod_and_etag(const fs::path& file)
{
// parse json at the beginning of the stream such as
// {"_url": "https://conda.anaconda.org/conda-forge/linux-64",
// "_etag": "W/\"6092e6a2b6cec6ea5aade4e177c3edda-8\"",
// "_mod": "Sat, 04 Apr 2020 03:29:49 GMT",
// "_cache_control": "public, max-age=1200"
auto extract_subjson = [](std::ifstream& s) {
char next;
std::string result;
bool escaped = false;
int i = 0, N = 4;
while (s.get(next))
{
if (next == '"')
{
if (!escaped)
{
i++;
}
else
{
escaped = false;
}
// 4 keys == 4 ticks
if (i == 4 * N)
{
return result + "\"}";
}
}
else if (next == '\\')
{
escaped = true;
}
result.push_back(next);
}
return std::string();
};
std::ifstream in_file = open_ifstream(file);
auto json = extract_subjson(in_file);
nlohmann::json result;
try
{
result = nlohmann::json::parse(json);
return result;
}
catch (...)
{
LOG_WARNING << "Could not parse mod/etag header";
return nlohmann::json();
}
}
std::string cache_fn_url(const std::string& url)
{
return cache_name_from_url(url) + ".json";

2
libmamba/tests/CMakeLists.txt
View File

@ -44,6 +44,8 @@ file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/env_file_test
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/validation_data
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/repodata_json_cache
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
target_link_libraries(test_libmamba PRIVATE GTest::GTest GTest::Main Threads::Threads)
target_link_libraries(test_libmamba PUBLIC libmamba)

53
libmamba/tests/repodata_json_cache/test_1.json Normal file
View File

@ -0,0 +1,53 @@
{"_cache_control":"","_etag":"","_mod":"Fri, 11 Feb 2022 13:52:44 GMT","_url":"file:///Users/wolfvollprecht/Programs/mamba/mamba/tests/channel_a/linux-64/repodata.json",
"info": {
"subdir": "linux-64"
},
"packages": {
"A_0.1.0.tar.bz2": {
"build": "abc",
"build_number": 0,
"depends": [],
"license": "BSD",
"license_family": "BSD",
"md5": "85107fc10154734ef34a5a75685be684",
"name": "a",
"sha256": "398831eff682d2c975b360d64656d8f475cbc1f1b6d0ee33d86285190e7ee4d1",
"size": 222503,
"subdir": "linux-64",
"timestamp": 1578950023135,
"version": "0.1.0"
},
"A_0.2.0.tar.bz2": {
"build": "abc",
"build_number": 0,
"depends": [],
"license": "BSD",
"license_family": "BSD",
"md5": "85107fc10154734ef34a5a75685be684",
"name": "a",
"sha256": "398831eff682d2c975b360d64656d8f475cbc1f1b6d0ee33d86285190e7ee4d1",
"size": 222503,
"subdir": "linux-64",
"timestamp": 1578950023135,
"version": "0.2.0"
},
"B_0.1.0.tar.bz2": {
"build": "abc",
"build_number": 0,
"depends": [
"a"
],
"license": "BSD",
"license_family": "BSD",
"md5": "85107fc10154734ef34a5a75685be684",
"name": "b",
"sha256": "398831eff682d2c975b360d64656d8f475cbc1f1b6d0ee33d86285190e7ee4d1",
"size": 222503,
"subdir": "linux-64",
"timestamp": 1578950023135,
"version": "0.1.0"
}
},
"removed": [],
"repodata_version": 1
}

5
libmamba/tests/repodata_json_cache/test_2.json Normal file
View File

@ -0,0 +1,5 @@
{"_mod":"Fri, 11 Feb 2022 13:52:44 GMT","_url":"file:///Users/wolfvollprecht/Programs/mamba/mamba/tests/channel_a/linux-64/repodata.json",
"info": {
"subdir": "linux-64"
}
}

5
libmamba/tests/repodata_json_cache/test_3.json Normal file
View File

@ -0,0 +1,5 @@
{
"info": {
"subdir": "linux-64"
}
}

5
libmamba/tests/repodata_json_cache/test_4.json Normal file
View File

@ -0,0 +1,5 @@
{"_cache_control":"{{}}\",,,\"","_etag":"\n\n\"\"randome ecx,,ssd\n,,\"","_mod":"Fri, 11 Feb 2022 13:52:44 GMT","_url":"file:///Users/wolfvollprecht/Programs/mamba/mamba/tests/channel_a/linux-64/repodata.json",
"info": {
"subdir": "linux-64"
}
}

5
libmamba/tests/repodata_json_cache/test_5.json Normal file
View File

@ -0,0 +1,5 @@
{"_mod":"Fri, 11 Feb 2022 13:52:44 GMT","_url":"file:///Users/wolfvollprecht/Programs/mamba/mamba/tests/channel_a/linux-64/repodata.json" ,
",,info": {
"subdir": "linux-64"
}
}

28
libmamba/tests/test_cpp.cpp
View File

@ -8,6 +8,7 @@
#include "mamba/core/history.hpp"
#include "mamba/core/link.hpp"
#include "mamba/core/match_spec.hpp"
#include "mamba/core/subdirdata.hpp"
namespace mamba
{
@ -501,4 +502,31 @@ namespace mamba
EXPECT_FALSE(fs::exists("emptytestfile"));
EXPECT_FALSE(lexists("emptytestfile"));
}
TEST(subdirdata, parse_mod_etag)
{
fs::path cache_folder = fs::path("repodata_json_cache");
auto j = detail::read_mod_and_etag(cache_folder / "test_1.json");
EXPECT_EQ(j["_mod"], "Fri, 11 Feb 2022 13:52:44 GMT");
EXPECT_EQ(j["_url"], "file:///Users/wolfvollprecht/Programs/mamba/mamba/tests/channel_a/linux-64/repodata.json");
j = detail::read_mod_and_etag(cache_folder / "test_2.json");
EXPECT_EQ(j["_mod"], "Fri, 11 Feb 2022 13:52:44 GMT");
EXPECT_EQ(j["_url"], "file:///Users/wolfvollprecht/Programs/mamba/mamba/tests/channel_a/linux-64/repodata.json");
j = detail::read_mod_and_etag(cache_folder / "test_5.json");
EXPECT_EQ(j["_mod"], "Fri, 11 Feb 2022 13:52:44 GMT");
EXPECT_EQ(j["_url"], "file:///Users/wolfvollprecht/Programs/mamba/mamba/tests/channel_a/linux-64/repodata.json");
j = detail::read_mod_and_etag(cache_folder / "test_4.json");
EXPECT_EQ(j["_cache_control"], "{{}}\",,,\"");
EXPECT_EQ(j["_etag"], "\n\n\"\"randome ecx,,ssd\n,,\"");
EXPECT_EQ(j["_mod"], "Fri, 11 Feb 2022 13:52:44 GMT");
EXPECT_EQ(j["_url"], "file:///Users/wolfvollprecht/Programs/mamba/mamba/tests/channel_a/linux-64/repodata.json");
j = detail::read_mod_and_etag(cache_folder / "test_3.json");
EXPECT_TRUE(j.empty());
// EXPECT_EQ(j["_mod"], "Fri, 11 Feb 2022 13:52:44 GMT");
// EXPECT_EQ(j["_url"], "file:///Users/wolfvollprecht/Programs/mamba/mamba/tests/channel_a/linux-64/repodata.json");
}
} // namespace mamba