folkslinux
/
mamba
mirror of https://github.com/mamba-org/mamba.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

impl of RepoChecker 

improve compatible or upgrade checks of spec
improve tests using gmock
update CMakeLists to use gmock on C++ tests
update CI steps to install gmock
This commit is contained in:
Adrien DELSALLE 2021-05-28 18:27:41 +02:00
parent 5bb4cf0f52
commit 056a71639f
No known key found for this signature in database
GPG Key ID: 639D9226C33B92BB
9 changed files with 595 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.github/workflows/main.yml vendored
View File

@ -39,7 +39,7 @@ jobs:
run: |
conda config --add channels conda-forge
conda config --set channel_priority strict
conda create -q -y -n mamba-tests python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cxx-compiler cmake gtest reproc-cpp yaml-cpp
conda create -q -y -n mamba-tests python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cxx-compiler cmake gtest gmock reproc-cpp yaml-cpp
env:
PYTHON_VERSION: ${{ matrix.python-version }}
- name: Install dependencies
@ -105,7 +105,7 @@ jobs:
export MAMBA_ROOT_PREFIX=~/mambaroot
export MAMBA_EXE=$(pwd)/micromamba
. $MAMBA_ROOT_PREFIX/etc/profile.d/mamba.sh
micromamba create -y -p ~/build_env pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cxx-compiler cmake gtest cpp-filesystem reproc-cpp yaml-cpp cli11 -c conda-forge
micromamba create -y -p ~/build_env pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cxx-compiler cmake gtest gmock cpp-filesystem reproc-cpp yaml-cpp cli11 -c conda-forge
env:
PYTHON_VERSION: ${{ matrix.python-version }}
- name: build tests
@ -157,7 +157,7 @@ jobs:
export MAMBA_ROOT_PREFIX=~/mambaroot
export MAMBA_EXE=$(pwd)/micromamba
. $MAMBA_ROOT_PREFIX/etc/profile.d/mamba.sh
micromamba create -y -p ~/build_env pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cxx-compiler cmake gtest cpp-filesystem reproc-cpp yaml-cpp pyyaml cli11 -c conda-forge
micromamba create -y -p ~/build_env pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cxx-compiler cmake gtest gmock cpp-filesystem reproc-cpp yaml-cpp pyyaml cli11 -c conda-forge
env:
PYTHON_VERSION: ${{ matrix.python-version }}
- name: build micromamba
@ -251,7 +251,7 @@ jobs:
run: |
conda config --add channels conda-forge
conda config --set channel_priority strict
conda install -n base -q -y vs2017_win-64 python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cmake gtest ninja reproc-cpp yaml-cpp cli11
conda install -n base -q -y vs2017_win-64 python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cmake gtest gmock ninja reproc-cpp yaml-cpp cli11
env:
PYTHON_VERSION: ${{ matrix.python-version }}
- name: Install dependencies
@ -296,7 +296,7 @@ jobs:
run: |
conda config --add channels conda-forge
conda config --set channel_priority strict
conda create -q -y -n mamba-tests vs2017_win-64 python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cmake gtest ninja reproc-cpp yaml-cpp cli11 pytest
conda create -q -y -n mamba-tests vs2017_win-64 python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cmake gtest gmock ninja reproc-cpp yaml-cpp cli11 pytest
env:
PYTHON_VERSION: ${{ matrix.python-version }}
- name: Run C++ tests Windows
@ -392,7 +392,7 @@ jobs:
run: |
conda config --add channels conda-forge
conda config --set channel_priority strict
conda create -q -y -n mamba-tests vs2017_win-64 python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cmake gtest ninja reproc-cpp yaml-cpp pyyaml cli11 pytest
conda create -q -y -n mamba-tests vs2017_win-64 python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cmake gtest gmock ninja reproc-cpp yaml-cpp pyyaml cli11 pytest
env:
PYTHON_VERSION: ${{ matrix.python-version }}
- name: micromamba python based tests
@ -433,7 +433,7 @@ jobs:
run: |
conda config --add channels conda-forge
conda config --set channel_priority strict
conda create -q -y -n mamba-tests vs2017_win-64 python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cmake gtest ninja reproc-cpp yaml-cpp pyyaml cli11 pytest
conda create -q -y -n mamba-tests vs2017_win-64 python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cmake gtest gmock ninja reproc-cpp yaml-cpp pyyaml cli11 pytest
env:
PYTHON_VERSION: ${{ matrix.python-version }}
- name: micromamba python based tests with pwsh
@ -472,7 +472,7 @@ jobs:
run: |
conda config --add channels conda-forge
conda config --set channel_priority strict
conda create -q -y -n mamba-tests vs2017_win-64 python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cmake gtest ninja reproc-cpp yaml-cpp pyyaml cli11 pytest
conda create -q -y -n mamba-tests vs2017_win-64 python=$PYTHON_VERSION pip pybind11 libsolv libsodium libarchive "libcurl=7.76.1=*_0" nlohmann_json cpp-filesystem conda cmake gtest gmock ninja reproc-cpp yaml-cpp pyyaml cli11 pytest
env:
PYTHON_VERSION: ${{ matrix.python-version }}
- name: micromamba python based tests

2
README.md
View File

@ -160,7 +160,7 @@ You will additionally need to install cmake and cli11 for micromamba:
mamba install -c conda-forge cli11 cmake
```
For the C++ tests, you need Google Tests installed (e.g. `mamba install gtest`).
For the C++ tests, you need Google Test and Google Mock installed (e.g. `mamba install gtest gmock`).
To build the program using CMake, the following lines need to be used:
```bash

1
environment-dev.yml
View File

@ -13,6 +13,7 @@ dependencies:
- libcurl 7.76.1 *_0
- cxx-compiler
- gtest
- gmock
- cpp-filesystem
- reproc-cpp
- yaml-cpp

4
include/mamba/core/util.hpp
View File

@ -165,6 +165,10 @@ namespace mamba
bool ends_with(const std::string_view& str, const std::string_view& suffix);
bool contains(const std::string_view& str, const std::string_view& sub_str);
bool any_starts_with(const std::vector<std::string_view>& str, const std::string_view& prefix);
bool starts_with_any(const std::string_view& str, const std::vector<std::string_view>& prefix);
std::string_view strip(const std::string_view& input);
std::string_view lstrip(const std::string_view& input);
std::string_view rstrip(const std::string_view& input);

38
include/mamba/core/validate.hpp
View File

@ -294,14 +294,15 @@ namespace validate
virtual std::string canonicalize(const json& j) const;
std::string compatible_starts_with() const;
std::string upgradable_starts_with() const;
std::string compatible_prefix() const;
std::vector<std::string> upgrade_prefix() const;
bool is_compatible(const fs::path& p) const;
bool is_compatible(const json& j) const;
bool is_compatible(const std::string& version) const;
bool is_upgradable(const json& j) const;
bool is_upgradable(const std::string& version) const;
bool is_upgrade(const json& j) const;
bool is_upgrade(const std::string& version) const;
virtual bool upgradable() const;
@ -402,7 +403,7 @@ namespace validate
/**
* 'root' role interface.
*/
class RootRole
class RootRole : public RoleBase
{
public:
std::unique_ptr<RootRole> update(fs::path path);
@ -414,11 +415,12 @@ namespace validate
const std::string& url) const = 0;
protected:
RootRole() = default;
RootRole(std::shared_ptr<SpecBase> spec);
private:
virtual json read_root_file(const fs::path& p, bool update = false) const = 0;
virtual std::size_t root_version() const = 0;
// virtual json read_root_file(const fs::path& p, bool update = false) const = 0;
// virtual std::size_t root_version() const = 0;
// virtual SpecBase* spec_impl() = 0;
virtual std::unique_ptr<RootRole> create_update(const json& j) = 0;
};
@ -431,6 +433,7 @@ namespace validate
class RepoIndexChecker
{
public:
virtual ~RepoIndexChecker() = default;
virtual void verify_index(const json& j) const = 0;
virtual void verify_index(const fs::path& p) const = 0;
@ -450,8 +453,8 @@ namespace validate
RepoChecker(const std::string& url, const fs::path& local_trusted_root);
// Fowarding to a ``RepoIndexChecker`` implementation
bool check(const json& j);
bool check(const fs::path& p);
void verify_index(const json& j);
void verify_index(const fs::path& p);
private:
std::string m_base_url;
@ -486,9 +489,7 @@ namespace validate
* TUF v1.0.17 §2.1.1
* https://theupdateframework.github.io/specification/latest/#root
*/
class RootImpl final
: public RootRole
, public RoleBase
class RootImpl final : public RootRole
{
public:
RootImpl(const fs::path& p);
@ -514,9 +515,6 @@ namespace validate
void set_defined_roles(std::map<std::string, Key> keys,
std::map<std::string, RoleKeys> roles);
json read_root_file(const fs::path& p, bool update = false) const override;
std::size_t root_version() const override;
};
}
@ -537,7 +535,6 @@ namespace validate
std::set<RoleSignature> signatures(const json& j) const override;
protected:
std::string canonicalize(const json& j) const override;
bool upgradable() const override;
};
@ -548,9 +545,7 @@ namespace validate
/**
* 'root' role implementation.
*/
class RootImpl final
: public RootRole
, public RoleBase
class RootImpl final : public RootRole
{
public:
RootImpl(const fs::path& p);
@ -583,9 +578,6 @@ namespace validate
std::set<std::string> optionally_defined_roles() const override;
void set_defined_roles(std::map<std::string, RolePubKeys> keys);
json read_root_file(const fs::path& p, bool update = false) const override;
std::size_t root_version() const override;
};

20
src/core/util.cpp
View File

@ -211,6 +211,26 @@ namespace mamba
return str.size() >= prefix.size() && 0 == str.compare(0, prefix.size(), prefix);
}
bool any_starts_with(const std::vector<std::string_view>& str, const std::string_view& prefix)
{
for (auto& s : str)
{
if (starts_with(s, prefix))
return true;
}
return false;
}
bool starts_with_any(const std::string_view& str, const std::vector<std::string_view>& prefix)
{
for (auto& p : prefix)
{
if (starts_with(str, p))
return true;
}
return false;
}
bool contains(const std::string_view& str, const std::string_view& sub_str)
{
return str.find(sub_str) != std::string::npos;

156
src/core/validate.cpp
View File

@ -426,38 +426,72 @@ namespace validate
return m_spec_version;
}
std::string SpecBase::compatible_starts_with() const
std::string SpecBase::compatible_prefix() const
{
auto split_spec_version = mamba::split(m_spec_version, ".", 2);
auto spec_version_major = std::stoi(split_spec_version[0]);
if (spec_version_major == 0)
{
return split_spec_version[0] + "." + split_spec_version[1] + ".";
return split_spec_version[0] + "." + split_spec_version[1];
}
else
{
return split_spec_version[0] + ".";
return split_spec_version[0];
}
}
std::string SpecBase::upgradable_starts_with() const
std::vector<std::string> SpecBase::upgrade_prefix() const
{
auto split_spec_version = mamba::split(m_spec_version, ".", 2);
auto spec_version_major = std::stoi(split_spec_version[0]);
auto spec_version_minor = std::stoi(split_spec_version[1]);
if (spec_version_major == 0)
{
return split_spec_version[0] + "." + std::to_string(spec_version_minor + 1) + ".";
// Return the most recent possible upgrade first
return { "1", split_spec_version[0] + "." + std::to_string(spec_version_minor + 1) };
}
else
{
return std::to_string(spec_version_major + 1) + ".";
return { std::to_string(spec_version_major + 1) };
}
}
bool SpecBase::is_compatible(const fs::path& p) const
{
std::regex name_re;
std::smatch matches;
std::size_t min_match_size;
std::string f_name = p.filename().string();
std::string f_spec_version_str, f_version_str, f_type, f_ext;
name_re = "^(?:[1-9]+\\d*.)?(?:sv([1-9]\\d*|0\\.[1-9]\\d*).)?(\\w+)\\.(\\w+)$";
min_match_size = 3;
if (std::regex_search(f_name, matches, name_re) && (min_match_size <= matches.size()))
{
f_spec_version_str = matches[1].str();
if (!f_spec_version_str.empty())
{
return is_compatible(matches[1].str() + ".");
}
else
{
std::ifstream i(p);
json j;
i >> j;
return is_compatible(j);
}
}
else
{
return false;
}
}
bool SpecBase::is_compatible(const std::string& version) const
{
return mamba::starts_with(version, compatible_starts_with());
return mamba::starts_with(version, compatible_prefix() + ".");
}
bool SpecBase::is_compatible(const json& j) const
@ -473,17 +507,25 @@ namespace validate
}
}
bool SpecBase::is_upgradable(const std::string& version) const
bool SpecBase::is_upgrade(const std::string& version) const
{
return mamba::starts_with(version, upgradable_starts_with());
auto upgrade_prefixes = upgrade_prefix();
std::vector<std::string_view> possible_upgrades;
for (auto& s : upgrade_prefixes)
{
s += ".";
possible_upgrades.push_back(s);
}
return mamba::starts_with_any(version, possible_upgrades);
}
bool SpecBase::is_upgradable(const json& j) const
bool SpecBase::is_upgrade(const json& j) const
{
auto spec_version = get_json_value(j);
if (!spec_version.empty())
{
return is_upgradable(spec_version);
return is_upgrade(spec_version);
}
else
{
@ -663,8 +705,8 @@ namespace validate
if (!current_sv.is_compatible(new_sv->version_str()))
{
LOG_ERROR << "Incompatible 'spec_version' found in 'root' metadata, should start with '"
<< current_sv.compatible_starts_with() << "' but is: '"
<< new_sv->version_str() << "'";
<< current_sv.compatible_prefix() << "' but is: '" << new_sv->version_str()
<< "'";
throw spec_version_error();
}
@ -681,7 +723,7 @@ namespace validate
{
if (!fs::exists(p))
{
LOG_ERROR << "File not found for 'root' update: " << p.string();
LOG_ERROR << "File not found for '" << type() << "' role: " << p.string();
throw role_file_error();
}
@ -744,7 +786,7 @@ namespace validate
{
auto new_spec_version_str = f_spec_version_str + ".";
if (update && spec_version().is_upgradable(new_spec_version_str)
if (update && spec_version().is_upgrade(new_spec_version_str)
&& !spec_version().upgradable())
{
LOG_ERROR << "Please check for a client update, unsupported spec version: '"
@ -752,7 +794,7 @@ namespace validate
throw spec_version_error();
}
else if (!((!update && spec_version().is_compatible(new_spec_version_str))
|| (update && spec_version().is_upgradable(new_spec_version_str))))
|| (update && spec_version().is_upgrade(new_spec_version_str))))
{
LOG_ERROR << "Invalid spec version specified in file name: '" << f_spec_version_str
<< "'";
@ -846,13 +888,36 @@ namespace validate
}
}
RootRole::RootRole(std::shared_ptr<SpecBase> spec)
: RoleBase("root", spec)
{
}
std::vector<fs::path> RootRole::possible_update_files()
{
return {};
auto new_v = std::to_string(version() + 1);
auto compat_spec = spec_impl()->compatible_prefix();
auto upgrade_spec = spec_impl()->upgrade_prefix();
std::vector<fs::path> files;
// upgrade first
for (auto& s : upgrade_spec)
{
files.push_back(
mamba::join(".", std::vector<std::string>({ new_v, "sv" + s, "root.json" })));
}
// compatible next
files.push_back(
mamba::join(".", std::vector<std::string>({ new_v, "sv" + compat_spec, "root.json" })));
// then finally undefined spec
files.push_back(mamba::join(".", std::vector<std::string>({ new_v, "root.json" })));
return files;
}
std::unique_ptr<RootRole> RootRole::update(fs::path path)
{
auto j = read_root_file(path, true);
auto j = read_json_file(path, true);
return update(j);
}
@ -872,9 +937,9 @@ namespace validate
// TUF spec 5.3.5 - Check for a rollback attack
// Version number has to be N+1
if (root_update->root_version() != (root_version() + 1))
if (root_update->version() != (version() + 1))
{
if (root_update->root_version() > (root_version() + 1))
if (root_update->version() > (version() + 1))
{
LOG_ERROR << "Invalid 'root' metadata version, should be exactly N+1";
throw role_metadata_error();
@ -915,16 +980,14 @@ namespace validate
}
RootImpl::RootImpl(const json& j)
: RootRole()
, RoleBase("root", std::make_shared<SpecImpl>())
: RootRole(std::make_shared<SpecImpl>())
{
load_from_json(j);
}
RootImpl::RootImpl(const fs::path& path)
: RootRole()
, RoleBase("root", std::make_shared<SpecImpl>())
: RootRole(std::make_shared<SpecImpl>())
{
auto j = read_json_file(path);
load_from_json(j);
@ -957,16 +1020,6 @@ namespace validate
return m_defined_roles.at("root");
}
json RootImpl::read_root_file(const fs::path& p, bool update) const
{
return read_json_file(p, update);
}
std::size_t RootImpl::root_version() const
{
return version();
}
std::set<std::string> RootImpl::mandatory_defined_roles() const
{
return { "root", "snapshot", "targets", "timestamp" };
@ -1099,15 +1152,13 @@ namespace validate
}
RootImpl::RootImpl(const json& j)
: RootRole()
, RoleBase("root", std::make_shared<SpecImpl>())
: RootRole(std::make_shared<SpecImpl>())
{
load_from_json(j);
}
RootImpl::RootImpl(const fs::path& path)
: RootRole()
, RoleBase("root", std::make_shared<SpecImpl>())
: RootRole(std::make_shared<SpecImpl>())
{
auto j = read_json_file(path);
load_from_json(j);
@ -1139,16 +1190,6 @@ namespace validate
check_role_signatures(j, *this);
}
json RootImpl::read_root_file(const fs::path& p, bool update) const
{
return read_json_file(p, update);
}
std::size_t RootImpl::root_version() const
{
return version();
}
json RootImpl::upgraded_signable() const
{
json v1_equivalent_root;
@ -1545,18 +1586,29 @@ namespace validate
p_index_checker = root->build_index_checker(fs::path(url));
};
void RepoChecker::verify_index(const json& j)
{
p_index_checker->verify_index(j);
}
void RepoChecker::verify_index(const fs::path& p)
{
p_index_checker->verify_index(p);
}
std::unique_ptr<RootRole> RepoChecker::get_root_role()
{
std::unique_ptr<RootRole> updated_root;
if (v06::SpecImpl().is_compatible(m_local_trusted_root.string()))
if (v06::SpecImpl().is_compatible(m_local_trusted_root))
{
updated_root = std::make_unique<v06::RootImpl>(m_local_trusted_root);
}
else if (v1::SpecImpl().is_compatible(m_local_trusted_root.string()))
else if (v1::SpecImpl().is_compatible(m_local_trusted_root))
{
return std::make_unique<v1::RootImpl>(m_local_trusted_root);
updated_root = std::make_unique<v1::RootImpl>(m_local_trusted_root);
}
else
{
LOG_ERROR << "Invalid spec version for 'root' initial trusted file";
throw spec_version_error();
@ -1567,6 +1619,7 @@ namespace validate
while (true)
{
fs::path p = "";
// Update from the most recent spec supported by this client
for (auto& f : update_files)
{
if (fs::exists(f))
@ -1582,6 +1635,7 @@ namespace validate
updated_root = updated_root->update(p);
update_files = updated_root->possible_update_files();
}
return updated_root;
};
} // namespace validate

9
test/CMakeLists.txt
View File

@ -1,13 +1,6 @@
cmake_minimum_required(VERSION 3.1)
if(WIN32)
find_package(GTest)
else()
add_library(GTest::GTest INTERFACE IMPORTED)
target_link_libraries(GTest::GTest INTERFACE gtest)
add_library(GTest::Main INTERFACE IMPORTED)
target_link_libraries(GTest::Main INTERFACE gtest_main)
endif()
find_package(GTest)
include_directories(${GTEST_INCLUDE_DIRS} SYSTEM)

567
test/test_validate.cpp
View File

File diff suppressed because it is too large Load Diff