mirror of https://github.com/mamba-org/mamba.git
[mamba-content-trust] Add integration test (#3234)
* -Fix trusted_channels in ctx not working with more than one item -Add test for mamba-content-trust (mamba client side) using conda-content-trust (server side) -Add missing install with pip (to fix) * Remove repo_signed (supposed to be generated with testserver_pkg_signing.sh) * Add MAMBA_ROOT_PREFIX to Taskfile.dist.yml
This commit is contained in:
Hind-M
GitHub
parent
4270dd400d
commit
cad6793e21
|
|
@ -163,3 +163,34 @@ jobs:
|
|||
unset CONDARC # Interferes with tests
|
||||
python -m pytest micromamba/tests/ \
|
||||
${{ runner.debug == 'true' && '-v --capture=tee-sys' || '--exitfirst' }}
|
||||
|
||||
verify_pkg_tests:
|
||||
name: mamba-content-trust tests
|
||||
needs: ["build_shared_unix"]
|
||||
runs-on: ${{ inputs.os }}
|
||||
if: startsWith(inputs.os, 'ubuntu')
|
||||
steps:
|
||||
- name: Checkout mamba repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Restore workspace
|
||||
uses: ./.github/actions/workspace
|
||||
with:
|
||||
action: restore
|
||||
path: build/
|
||||
key_suffix: ${{ inputs.os }}-${{ inputs.build_type }}
|
||||
- name: Create build environment
|
||||
uses: mamba-org/setup-micromamba@v1
|
||||
with:
|
||||
environment-file: ./build/environment.lock
|
||||
environment-name: build_env
|
||||
- name: Run tests using conda-content-trust (server side)
|
||||
shell: bash -l {0} -euo pipefail -x
|
||||
run: |
|
||||
export TEST_MAMBA_EXE=$(pwd)/build/micromamba/mamba
|
||||
export MAMBA_ROOT_PREFIX="${HOME}/micromamba"
|
||||
unset CONDARC # Interferes with tests
|
||||
# FIXME this is apparently a bug to be fixed (pip specs are not installed from env yml file)
|
||||
python -m pip install securesystemslib --no-input --no-deps
|
||||
cd micromamba/test-server
|
||||
./generate_gpg_keys.sh
|
||||
./testserver_pkg_signing.sh
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ vars:
|
|||
DOCS_DOXYGEN_XML_DIR: '{{.PWD}}/{{.DOCS_DIR}}/doxygen-xml'
|
||||
MAMBA_NAME: 'mamba' # Depend on preset...
|
||||
TEST_MAMBA_EXE: '{{.PWD}}/{{.CMAKE_BUILD_DIR}}/micromamba/{{.MAMBA_NAME}}'
|
||||
MAMBA_ROOT_PREFIX: '${HOME}/micromamba'
|
||||
CPU_PERCENTAGE: 75
|
||||
CPU_TOTAL:
|
||||
sh: >-
|
||||
|
|
@ -222,6 +223,7 @@ tasks:
|
|||
deps: [{task: '_build', vars: {target: '{{.MAMBA_NAME'}}]
|
||||
env:
|
||||
TEST_MAMBA_EXE: '{{.TEST_MAMBA_EXE}}'
|
||||
MAMBA_ROOT_PREFIX: '{{.MAMBA_ROOT_PREFIX}}'
|
||||
# Explicitly using this as var since env does not override shell environment
|
||||
vars:
|
||||
GNUPGHOME: '{{.BUILD_DIR}}/gnupg'
|
||||
|
|
|
|||
|
|
@ -39,10 +39,11 @@ namespace mamba
|
|||
bool extra_safety_checks = false;
|
||||
bool verify_artifacts = false;
|
||||
|
||||
// TODO test with multiple channels in there and check behavior
|
||||
// i.e uncommenting `conda-forge` when possible
|
||||
// and removing "http://127.0.0.1:8000/get/channel0" (should only be in integration tests)
|
||||
// Should we consider removing this and use `channels` instead?
|
||||
// TODO Uncomment `conda-forge` or whatever trusted_channels when possible
|
||||
// (i.e server side package signing ready)
|
||||
// Remove "http://127.0.0.1:8000/get/channel0"
|
||||
// (should only be used in integration tests,
|
||||
// this one is for testing with quetz)
|
||||
std::vector<std::string> trusted_channels = {
|
||||
/*"conda-forge", */ "http://127.0.0.1:8000/get/channel0"
|
||||
};
|
||||
|
|
|
|||
|
|
@ -32,11 +32,11 @@ namespace mamba
|
|||
|
||||
explicit RepoCheckerStore(repo_checker_list checkers);
|
||||
|
||||
[[nodiscard]] auto find_checker(const Channel& chan) const -> const RepoChecker*;
|
||||
[[nodiscard]] auto find_checker(const Channel& chan) -> RepoChecker*;
|
||||
|
||||
[[nodiscard]] auto contains_checker(const Channel& chan) const -> bool;
|
||||
[[nodiscard]] auto contains_checker(const Channel& chan) -> bool;
|
||||
|
||||
[[nodiscard]] auto at_checker(const Channel& chan) const -> const RepoChecker&;
|
||||
[[nodiscard]] auto at_checker(const Channel& chan) -> RepoChecker&;
|
||||
|
||||
private:
|
||||
|
||||
|
|
|
|||
|
|
@ -42,7 +42,6 @@ namespace mamba
|
|||
|
||||
// Initialization
|
||||
fs::create_directories(checker.cache_path());
|
||||
checker.generate_index_checker();
|
||||
|
||||
repo_checkers.emplace_back(std::move(chan), std::move(checker));
|
||||
}
|
||||
|
|
@ -55,7 +54,7 @@ namespace mamba
|
|||
{
|
||||
}
|
||||
|
||||
auto RepoCheckerStore::find_checker(const Channel& chan) const -> const RepoChecker*
|
||||
auto RepoCheckerStore::find_checker(const Channel& chan) -> RepoChecker*
|
||||
{
|
||||
for (auto& [candidate_chan, checker] : m_repo_checkers)
|
||||
{
|
||||
|
|
@ -67,12 +66,12 @@ namespace mamba
|
|||
return nullptr;
|
||||
}
|
||||
|
||||
auto RepoCheckerStore::contains_checker(const Channel& chan) const -> bool
|
||||
auto RepoCheckerStore::contains_checker(const Channel& chan) -> bool
|
||||
{
|
||||
return find_checker(chan) != nullptr;
|
||||
}
|
||||
|
||||
auto RepoCheckerStore::at_checker(const Channel& chan) const -> const RepoChecker&
|
||||
auto RepoCheckerStore::at_checker(const Channel& chan) -> RepoChecker&
|
||||
{
|
||||
if (auto ptr = find_checker(chan))
|
||||
{
|
||||
|
|
|
|||
|
|
@ -592,6 +592,7 @@ namespace mamba
|
|||
if (repo_checker)
|
||||
{
|
||||
LOG_INFO << "RepoChecker successfully created.";
|
||||
repo_checker->generate_index_checker();
|
||||
repo_checker->verify_package(
|
||||
pkg.json_signable(),
|
||||
std::string_view(pkg.signatures)
|
||||
|
|
@ -600,6 +601,10 @@ namespace mamba
|
|||
else
|
||||
{
|
||||
LOG_ERROR << "Could not create a valid RepoChecker.";
|
||||
throw std::runtime_error(fmt::format(
|
||||
R"(Could not verify "{}". Please make sure the package signatures are available and 'trusted-channels' are configured correctly. Alternatively, try downloading without '--verify-artifacts' flag.)",
|
||||
pkg.name
|
||||
));
|
||||
}
|
||||
}
|
||||
LOG_INFO << "'" << pkg.name << "' trusted from '" << pkg.channel << "'";
|
||||
|
|
|
|||
|
|
@ -8,14 +8,12 @@
|
|||
"build_number": 0,
|
||||
"depends": [],
|
||||
"license": "BSD",
|
||||
"license_family": "BSD",
|
||||
"md5": "2a8595f37faa2950e1b433acbe91d481",
|
||||
"md5": "719449904d9d4ac9853437a9504f87c5",
|
||||
"name": "test-package",
|
||||
"noarch": "generic",
|
||||
"sha256": "b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f6988",
|
||||
"size": 5719,
|
||||
"sha256": "1c8942fd0ad0dd3f780bff1a159a6ff8b82968965a630b3e1bff27b7168f68b5",
|
||||
"size": 5747,
|
||||
"subdir": "noarch",
|
||||
"timestamp": 1613117294885,
|
||||
"timestamp": 1613117294,
|
||||
"version": "0.1"
|
||||
}
|
||||
},
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -8,14 +8,12 @@
|
|||
"build_number": 0,
|
||||
"depends": [],
|
||||
"license": "BSD",
|
||||
"license_family": "BSD",
|
||||
"md5": "2a8595f37faa2950e1b433acbe91d481",
|
||||
"md5": "719449904d9d4ac9853437a9504f87c5",
|
||||
"name": "test-package",
|
||||
"noarch": "generic",
|
||||
"sha256": "b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f6988",
|
||||
"size": 5719,
|
||||
"sha256": "1c8942fd0ad0dd3f780bff1a159a6ff8b82968965a630b3e1bff27b7168f68b5",
|
||||
"size": 5747,
|
||||
"subdir": "noarch",
|
||||
"timestamp": 1613117294885,
|
||||
"timestamp": 1613117294,
|
||||
"version": "0.1"
|
||||
}
|
||||
},
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -8,14 +8,12 @@
|
|||
"build_number": 0,
|
||||
"depends": [],
|
||||
"license": "BSD",
|
||||
"license_family": "BSD",
|
||||
"md5": "2a8595f37faa2950e1b433acbe91d481",
|
||||
"md5": "719449904d9d4ac9853437a9504f87c5",
|
||||
"name": "test-package",
|
||||
"noarch": "generic",
|
||||
"sha256": "b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f6988",
|
||||
"size": 5719,
|
||||
"sha256": "1c8942fd0ad0dd3f780bff1a159a6ff8b82968965a630b3e1bff27b7168f68b5",
|
||||
"size": 5747,
|
||||
"subdir": "noarch",
|
||||
"timestamp": 1613117294885,
|
||||
"timestamp": 1613117294,
|
||||
"version": "0.1"
|
||||
}
|
||||
},
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
|
|
@ -1,29 +0,0 @@
|
|||
{
|
||||
"signatures": {
|
||||
"350689d2a4dff47a6c260be35dca2025f85a971695a13b25de5bd40d7801bbb0": {
|
||||
"other_headers": "04001608001d162104860d71890caeafcbda51ce4539ccaa003193e2090502653a7512",
|
||||
"signature": "b5a94a6d610763e6322b65550bb0f2c6f2c828150ece9b67d22511ca2569b6266e43393abe34a347072c689679c2f4ef3fb38c8325966049c9cfa9d3d6dcd702"
|
||||
}
|
||||
},
|
||||
"signed": {
|
||||
"delegations": {
|
||||
"key_mgr": {
|
||||
"pubkeys": [
|
||||
"013ddd714962866d12ba5bae273f14d48c89cf0773dee2dbf6d4561e521c83f7"
|
||||
],
|
||||
"threshold": 1
|
||||
},
|
||||
"root": {
|
||||
"pubkeys": [
|
||||
"350689d2a4dff47a6c260be35dca2025f85a971695a13b25de5bd40d7801bbb0"
|
||||
],
|
||||
"threshold": 1
|
||||
}
|
||||
},
|
||||
"expiration": "2024-10-25T14:17:54Z",
|
||||
"metadata_spec_version": "0.6.0",
|
||||
"timestamp": "2023-10-26T14:17:54Z",
|
||||
"type": "root",
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
{
|
||||
"signatures": {
|
||||
"013ddd714962866d12ba5bae273f14d48c89cf0773dee2dbf6d4561e521c83f7": {
|
||||
"signature": "0a4ee7615fb9115e5bbd62a1a7f33455e6c26d46e0e815a6015258ad17d41570c068444311eb165828670cc4275b5650fb03eb235d10062b6d98f2650d62c50b"
|
||||
}
|
||||
},
|
||||
"signed": {
|
||||
"delegations": {
|
||||
"pkg_mgr": {
|
||||
"pubkeys": [
|
||||
"f46b5a7caa43640744186564c098955147daa8bac4443887bc64d8bfee3d3569"
|
||||
],
|
||||
"threshold": 1
|
||||
}
|
||||
},
|
||||
"expiration": "2024-10-25T14:17:54Z",
|
||||
"metadata_spec_version": "0.6.0",
|
||||
"timestamp": "2023-10-26T14:17:54Z",
|
||||
"type": "key_mgr",
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
{
|
||||
"info": {
|
||||
"subdir": "noarch"
|
||||
},
|
||||
"packages": {
|
||||
"test-package-0.1-0.tar.bz2": {
|
||||
"build": "0",
|
||||
"build_number": 0,
|
||||
"depends": [],
|
||||
"license": "BSD",
|
||||
"license_family": "BSD",
|
||||
"md5": "2a8595f37faa2950e1b433acbe91d481",
|
||||
"name": "test-package",
|
||||
"noarch": "generic",
|
||||
"sha256": "b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f6988",
|
||||
"size": 5719,
|
||||
"subdir": "noarch",
|
||||
"timestamp": 1613117294885,
|
||||
"version": "0.1"
|
||||
}
|
||||
},
|
||||
"packages.conda": {},
|
||||
"removed": [],
|
||||
"repodata_version": 1,
|
||||
"signatures": {
|
||||
"test-package-0.1-0.tar.bz2": {
|
||||
"f46b5a7caa43640744186564c098955147daa8bac4443887bc64d8bfee3d3569": {
|
||||
"signature": "0a50063539baf249970f1d08b07f00f544e2d87982826790e9ec6e80874ad90aec21a9607cf38bb58897163533c39cb4a4f1c741a7f8e9e4f67e2ff2087d2d00"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -89,8 +89,10 @@ class RepoSigner:
|
|||
self.folder.mkdir(exist_ok=True)
|
||||
self.create_root(self.keys)
|
||||
self.create_key_mgr(self.keys)
|
||||
self.create_pkg_mgr(self.keys)
|
||||
for f in glob.glob(str(self.in_folder / "**" / "repodata.json")):
|
||||
self.sign_repodata(Path(f), self.keys)
|
||||
self.copy_signing_root_file()
|
||||
return self.folder
|
||||
|
||||
def create_root(self, keys):
|
||||
|
|
@ -168,10 +170,53 @@ class RepoSigner:
|
|||
# Doing delegation processing.
|
||||
cct_authentication.verify_delegation("key_mgr", key_mgr_metadata, root_metadata)
|
||||
|
||||
print("[reposigner] success: key mgr metadata verified based on root metadata.")
|
||||
print("[reposigner] Success: key_mgr metadata verified based on root metadata.")
|
||||
|
||||
return key_mgr
|
||||
|
||||
# Adding this to be compatible with `mamba` (in `conda` they don't seem to have the `pkg_mgr.json` file)
|
||||
# But the signing does use delegation to `pkg_mgr` role in both cases
|
||||
def create_pkg_mgr(self, keys):
|
||||
private_key_pkg_mgr = cct_common.PrivateKey.from_hex(keys["pkg_mgr"][0]["private"])
|
||||
pkg_mgr = cct_metadata_construction.build_delegating_metadata(
|
||||
metadata_type="pkg_mgr",
|
||||
delegations=None,
|
||||
version=1,
|
||||
# timestamp default: now
|
||||
# expiration default: now plus root expiration default duration
|
||||
)
|
||||
|
||||
pkg_mgr = cct_signing.wrap_as_signable(pkg_mgr)
|
||||
|
||||
# sign dictionary in place
|
||||
cct_signing.sign_signable(pkg_mgr, private_key_pkg_mgr)
|
||||
|
||||
pkg_mgr_serialized = cct_common.canonserialize(pkg_mgr)
|
||||
with open(self.folder / "pkg_mgr.json", "wb") as fobj:
|
||||
fobj.write(pkg_mgr_serialized)
|
||||
|
||||
# let's run a verification
|
||||
key_mgr_metadata = cct_common.load_metadata_from_file(self.folder / "key_mgr.json")
|
||||
pkg_mgr_metadata = cct_common.load_metadata_from_file(self.folder / "pkg_mgr.json")
|
||||
|
||||
cct_common.checkformat_signable(key_mgr_metadata)
|
||||
|
||||
if "delegations" not in key_mgr_metadata["signed"]:
|
||||
raise ValueError('Expected "delegations" entry in key_mgr metadata.')
|
||||
|
||||
key_mgr_delegations = key_mgr_metadata["signed"]["delegations"] # for brevity
|
||||
cct_common.checkformat_delegations(key_mgr_delegations)
|
||||
if "pkg_mgr" not in key_mgr_delegations:
|
||||
raise ValueError('Missing expected delegation to "pkg_mgr" in key_mgr metadata.')
|
||||
cct_common.checkformat_delegation(key_mgr_delegations["pkg_mgr"])
|
||||
|
||||
# Doing delegation processing.
|
||||
cct_authentication.verify_delegation("pkg_mgr", pkg_mgr_metadata, key_mgr_metadata)
|
||||
|
||||
print("[reposigner] Success: pkg_mgr metadata verified based on key_mgr metadata.")
|
||||
|
||||
return pkg_mgr
|
||||
|
||||
def sign_repodata(self, repodata_fn, keys):
|
||||
target_folder = self.folder / repodata_fn.parent.name
|
||||
if not target_folder.exists():
|
||||
|
|
@ -185,6 +230,31 @@ class RepoSigner:
|
|||
cct_signing.sign_all_in_repodata(str(final_fn), pkg_mgr_key)
|
||||
print(f"[reposigner] Signed {final_fn}")
|
||||
|
||||
# Copy actual 'test-package-0.1-0.tar.bz2' to serving directory ('repo_signed')
|
||||
pkg_bz2_src_fn = repodata_fn.parent / "test-package-0.1-0.tar.bz2"
|
||||
pkg_bz2_dst_fn = target_folder / "test-package-0.1-0.tar.bz2"
|
||||
print("copy", pkg_bz2_src_fn, pkg_bz2_dst_fn)
|
||||
shutil.copyfile(pkg_bz2_src_fn, pkg_bz2_dst_fn)
|
||||
print("[reposigner] 'test-package-0.1-0.tar.bz2' copied")
|
||||
|
||||
def copy_signing_root_file(self):
|
||||
# Copy root json file to 'ref_path'
|
||||
# as this should be available in a safe place locally
|
||||
root_prefix = Path(os.environ["MAMBA_ROOT_PREFIX"])
|
||||
if not root_prefix:
|
||||
fatal_error("MAMBA_ROOT_PREFIX is not set!")
|
||||
|
||||
# '7da7dc10' corresponds to id of channel 'http://localhost:8000/mychannel'
|
||||
channel_initial_trusted_root_role = root_prefix / "etc/trusted-repos/7da7dc10"
|
||||
if not channel_initial_trusted_root_role.exists():
|
||||
os.makedirs(channel_initial_trusted_root_role)
|
||||
|
||||
shutil.copy(
|
||||
self.folder / "1.root.json",
|
||||
channel_initial_trusted_root_role / "root.json",
|
||||
)
|
||||
print("Initial trusted root copied")
|
||||
|
||||
|
||||
class ChannelHandler(SimpleHTTPRequestHandler):
|
||||
url_pattern = re.compile(r"^/(?:t/[^/]+/)?([^/]+)")
|
||||
|
|
@ -219,6 +289,12 @@ class ChannelHandler(SimpleHTTPRequestHandler):
|
|||
|
||||
self.send_response(404)
|
||||
|
||||
def do_HEAD(self) -> None:
|
||||
if self.path.endswith("_mgr.json"):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-type", "text/html")
|
||||
self.end_headers()
|
||||
|
||||
def basic_do_HEAD(self) -> None:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-type", "text/html")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,53 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail -x
|
||||
|
||||
# Directory of this file
|
||||
readonly __DIR__="$(cd "$(dirname "${BASH_SOURCE[0]:?}")" && pwd)"
|
||||
# reposerver python script
|
||||
readonly reposerver="${__DIR__}/reposerver.py"
|
||||
# Conda mock repository
|
||||
readonly repo="${__DIR__}/repo/"
|
||||
|
||||
# Default value "mamba"
|
||||
export TEST_MAMBA_EXE="${TEST_MAMBA_EXE:-micromamba/mamba}"
|
||||
|
||||
# Avoid externally configured .condarc file
|
||||
unset CONDARC
|
||||
|
||||
# Set up a temporary space for Conda environment and packages
|
||||
readonly test_dir="$(mktemp -d -t mamba-test-reposerver-XXXXXXXXXX)"
|
||||
export CONDA_ENVS_DIRS="${test_dir}/envs"
|
||||
export CONDA_PKGS_DIRS="${test_dir}/pkgs"
|
||||
readonly this_pid="$$"
|
||||
# On exit, kill all subprocess and cleanup test directory.
|
||||
trap 'rm -rf "${test_dir}"; pkill -P ${this_pid} || true' EXIT
|
||||
|
||||
start_server() {
|
||||
exec python "${reposerver}" -n mychannel -d "${repo}" "$@"
|
||||
}
|
||||
|
||||
test_install() {
|
||||
local tmp=$(mktemp -d)
|
||||
local condarc="${tmp}/condarc"
|
||||
"${TEST_MAMBA_EXE}" create -y -p "${tmp}/env1" --override-channels -c $1/mychannel test-package -vvv --repodata-ttl=0 --trusted-channels "http://localhost:8000/mychannel" --verify-artifacts
|
||||
}
|
||||
|
||||
check_dwd_pkg() {
|
||||
if [ -e "${test_dir}/pkgs/test-package-0.1-0.tar.bz2" ]; then
|
||||
echo -e "\e[32mtest-package-0.1-0.tar.bz2 successfully verified and downloaded!\e[0m"
|
||||
else
|
||||
echo -e "\e[31mtest-package-0.1-0.tar.bz2 does not exist!\e[0m"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ "$(uname -s)" == "Linux" ]]; then
|
||||
export KEY1=$(gpg --fingerprint "MAMBA1")
|
||||
export KEY2=$(gpg --fingerprint "MAMBA2")
|
||||
|
||||
start_server --auth none --sign & PID=$!
|
||||
test_install http://localhost:8000
|
||||
check_dwd_pkg
|
||||
kill -TERM $PID
|
||||
fi
|
||||
Loading…
Reference in New Issue