mirror of https://github.com/mamba-org/mamba.git
Add support for configuring proxies in .condarc (#1814)
* Add support for proxy settings in .condarc This commit is based on https://github.com/mamba-org/mamba/pull/161/ * Add test for proxy configuration * Remove env variables from proxy servers configurable They are already handled by curl * Add tests for proxy_match * Remove and improve debug messages * Use EXPECT_STREQ for proxy match tests * Use std:string instead of char* * Add micromamba proxy tests * Retrigger CI * Try to fix proxy tests in CI * Add missing file and fix executable name * Try to fix windows build * Use install xtensor package for proxy test * Try to fix tests on windows * Pass proxy settings between libmamba and mamba * Use long form of `--script` parameter for mitmproxy Co-authored-by: Jonas Haag <jonas@lophus.org> * Clarify variable name Co-authored-by: Jonas Haag <jonas@lophus.org> * Add docstring to dump_proxy_connections.py * Add explanation to micromamba proxy test * Cleanup proxy_match function * Redact passwords in proxy urls when logging * Renamed redact_url to redact_url_password * Revert "Renamed redact_url to redact_url_password" This reverts commitca39afda21. * Revert "Redact passwords in proxy urls when logging" This reverts commit270eb70420. * Use existing hide_secrets function to hide password in proxy url * Rewrite proxy_match function to be in line with conda Co-authored-by: Jonas Haag <jonas@lophus.org>
This commit is contained in:
Adrian Freund
GitHub
parent
b9413201d8
commit
02e4385d2f
|
|
@ -187,6 +187,7 @@ namespace mamba
|
|||
int retry_backoff = 3; // retry_timeout * retry_backoff
|
||||
int max_retries = 3; // max number of retries
|
||||
|
||||
std::map<std::string, std::string> proxy_servers;
|
||||
// ssl verify can be either an empty string (regular SSL verification),
|
||||
// the string "<false>" to indicate no SSL verification, or a path to
|
||||
// a directory with cert files, or a cert file.
|
||||
|
|
@ -236,7 +237,7 @@ namespace mamba
|
|||
// usernames on anaconda.org can have a underscore, which influences the
|
||||
// first two characters
|
||||
const std::regex token_regex{ "/t/([a-zA-Z0-9-_]{0,2}[a-zA-Z0-9-]*)" };
|
||||
const std::regex http_basicauth_regex{ "://([^\\s]+):([^\\s]+)@" };
|
||||
const std::regex http_basicauth_regex{ "(://|^)([^\\s]+):([^\\s]+)@" };
|
||||
const std::regex scheme_regex{ "[a-z][a-z0-9]{0,11}://" };
|
||||
|
||||
static Context& instance();
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@
|
|||
#include <time.h>
|
||||
#include <vector>
|
||||
#include <chrono>
|
||||
#include <optional>
|
||||
|
||||
#if defined(__PPC64__) || defined(__ppc64__) || defined(_ARCH_PPC64)
|
||||
#include <iomanip>
|
||||
|
|
@ -226,6 +227,8 @@ namespace mamba
|
|||
return ends_with(filename, ".yml") || ends_with(filename, ".yaml");
|
||||
}
|
||||
|
||||
std::optional<std::string> proxy_match(const std::string& url);
|
||||
|
||||
} // namespace mamba
|
||||
|
||||
#endif // MAMBA_UTIL_HPP
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@
|
|||
#include "mamba/core/fsutil.hpp"
|
||||
#include "mamba/core/output.hpp"
|
||||
#include "mamba/core/transaction.hpp"
|
||||
#include "mamba/core/url.hpp"
|
||||
|
||||
namespace mamba
|
||||
{
|
||||
|
|
@ -1182,6 +1183,16 @@ namespace mamba
|
|||
.needs({ "cacert_path", "offline" })
|
||||
.set_post_merge_hook(detail::ssl_verify_hook));
|
||||
|
||||
insert(Configurable("proxy_servers", &ctx.proxy_servers)
|
||||
.group("Network")
|
||||
.set_rc_configurable()
|
||||
.description("Use a proxy server for network connections")
|
||||
.long_description(unindent(R"(
|
||||
'proxy_servers' should be a dictionary where the key is either in the form of
|
||||
scheme://hostname or just a scheme for which the proxy server should be used and
|
||||
the value is the url of the proxy server, optionally with username and password
|
||||
in the form of scheme://username:password@hostname.)")));
|
||||
|
||||
// Solver
|
||||
insert(Configurable("channel_priority", &ctx.channel_priority)
|
||||
.group("Solver")
|
||||
|
|
|
|||
|
|
@ -260,6 +260,13 @@ namespace mamba
|
|||
curl_easy_setopt(handle, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NO_REVOKE);
|
||||
}
|
||||
|
||||
std::optional<std::string> proxy = proxy_match(url);
|
||||
if (proxy)
|
||||
{
|
||||
curl_easy_setopt(handle, CURLOPT_PROXY, proxy->c_str());
|
||||
LOG_INFO << "Using Proxy " << hide_secrets(*proxy);
|
||||
}
|
||||
|
||||
std::string& ssl_verify = Context::instance().ssl_verify;
|
||||
if (ssl_verify.size())
|
||||
{
|
||||
|
|
@ -267,11 +274,20 @@ namespace mamba
|
|||
{
|
||||
curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L);
|
||||
curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L);
|
||||
if (proxy)
|
||||
{
|
||||
curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYPEER, 0L);
|
||||
curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYHOST, 0L);
|
||||
}
|
||||
}
|
||||
else if (ssl_verify == "<system>")
|
||||
{
|
||||
#ifdef LIBMAMBA_STATIC_DEPS
|
||||
curl_easy_setopt(handle, CURLOPT_CAINFO, nullptr);
|
||||
if (proxy)
|
||||
{
|
||||
curl_easy_setopt(handle, CURLOPT_PROXY_CAINFO, nullptr);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
else
|
||||
|
|
@ -283,6 +299,10 @@ namespace mamba
|
|||
else
|
||||
{
|
||||
curl_easy_setopt(handle, CURLOPT_CAINFO, ssl_verify.c_str());
|
||||
if (proxy)
|
||||
{
|
||||
curl_easy_setopt(handle, CURLOPT_PROXY_CAINFO, ssl_verify.c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -182,11 +182,7 @@ namespace mamba
|
|||
copy = std::regex_replace(copy, Context::instance().token_regex, "/t/*****");
|
||||
}
|
||||
|
||||
if (contains(str, "://"))
|
||||
{
|
||||
copy = std::regex_replace(
|
||||
copy, Context::instance().http_basicauth_regex, "://$1:*****@");
|
||||
}
|
||||
copy = std::regex_replace(copy, Context::instance().http_basicauth_regex, "$1$2:*****@");
|
||||
|
||||
return copy;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ extern "C"
|
|||
#include <iomanip>
|
||||
#include <mutex>
|
||||
#include <condition_variable>
|
||||
#include <optional>
|
||||
#include <openssl/evp.h>
|
||||
|
||||
#include "mamba/core/environment.hpp"
|
||||
|
|
@ -46,6 +47,7 @@ extern "C"
|
|||
#include "mamba/core/util_os.hpp"
|
||||
#include "mamba/core/util_random.hpp"
|
||||
#include "mamba/core/fsutil.hpp"
|
||||
#include "mamba/core/url.hpp"
|
||||
|
||||
namespace mamba
|
||||
{
|
||||
|
|
@ -1478,4 +1480,43 @@ namespace mamba
|
|||
return std::string((const char*) output.data());
|
||||
}
|
||||
|
||||
std::optional<std::string> proxy_match(const std::string& url)
|
||||
{
|
||||
/* This is a reimplementation of requests.utils.select_proxy(), of the python requests
|
||||
library used by conda */
|
||||
auto& proxies = Context::instance().proxy_servers;
|
||||
if (proxies.empty())
|
||||
{
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
auto handler = URLHandler(url);
|
||||
auto scheme = handler.scheme();
|
||||
auto host = handler.host();
|
||||
std::vector<std::string> options;
|
||||
|
||||
if (host.empty())
|
||||
{
|
||||
options = {
|
||||
scheme,
|
||||
"all",
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
options = { scheme + "://" + host, scheme, "all://" + host, "all" };
|
||||
}
|
||||
|
||||
for (auto& option : options)
|
||||
{
|
||||
auto proxy = proxies.find(option);
|
||||
if (proxy != proxies.end())
|
||||
{
|
||||
return proxy->second;
|
||||
}
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
} // namespace mamba
|
||||
|
|
|
|||
|
|
@ -642,6 +642,23 @@ namespace mamba
|
|||
load_test_config("cacert_path:\nssl_verify: true"); // reset ssl verify to default
|
||||
}
|
||||
|
||||
TEST_F(Configuration, proxy_servers)
|
||||
{
|
||||
std::string rc = unindent(R"(
|
||||
proxy_servers:
|
||||
http: foo
|
||||
https: bar)");
|
||||
load_test_config(rc);
|
||||
auto& actual = config.at("proxy_servers").value<std::map<std::string, std::string>>();
|
||||
std::map<std::string, std::string> expected = { { "http", "foo" }, { "https", "bar" } };
|
||||
EXPECT_EQ(actual, expected);
|
||||
EXPECT_EQ(ctx.proxy_servers, expected);
|
||||
|
||||
EXPECT_EQ(config.sources().size(), 1);
|
||||
EXPECT_EQ(config.valid_sources().size(), 1);
|
||||
EXPECT_EQ(config.dump(), "proxy_servers:\n http: foo\n https: bar");
|
||||
}
|
||||
|
||||
TEST_F(Configuration, platform)
|
||||
{
|
||||
EXPECT_EQ(ctx.platform, ctx.host_platform);
|
||||
|
|
|
|||
|
|
@ -254,6 +254,12 @@ namespace mamba
|
|||
EXPECT_EQ(
|
||||
res,
|
||||
"http://root:*****@myweb.com/test.repo\nhttp://myweb.com/t/*****/test.repo http://myweb.com/t/*****/test.repo http://root:*****@myweb.com/test.repo");
|
||||
|
||||
res = Console::instance().hide_secrets("myweb.com/t/my-12345-token/test.repo");
|
||||
EXPECT_EQ(res, "myweb.com/t/*****/test.repo");
|
||||
|
||||
res = Console::instance().hide_secrets("root:secretpassword@myweb.com/test.repo");
|
||||
EXPECT_EQ(res, "root:*****@myweb.com/test.repo");
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@
|
|||
#include "mamba/core/mamba_fs.hpp"
|
||||
#include "mamba/core/util_scope.hpp"
|
||||
#include "mamba/core/fsutil.hpp"
|
||||
#include "mamba/core/context.hpp"
|
||||
|
||||
|
||||
namespace mamba
|
||||
{
|
||||
|
|
@ -139,4 +141,32 @@ namespace mamba
|
|||
EXPECT_TRUE(path::is_writable(existing_file_path));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(utils, proxy_match)
|
||||
{
|
||||
Context::instance().proxy_servers = { { "http", "foo" },
|
||||
{ "https", "bar" },
|
||||
{ "https://example.net", "foobar" },
|
||||
{ "all://example.net", "baz" },
|
||||
{ "all", "other" } };
|
||||
|
||||
EXPECT_EQ(*proxy_match("http://example.com/channel"), "foo");
|
||||
EXPECT_EQ(*proxy_match("http://example.net/channel"), "foo");
|
||||
EXPECT_EQ(*proxy_match("https://example.com/channel"), "bar");
|
||||
EXPECT_EQ(*proxy_match("https://example.com:8080/channel"), "bar");
|
||||
EXPECT_EQ(*proxy_match("https://example.net/channel"), "foobar");
|
||||
EXPECT_EQ(*proxy_match("ftp://example.net/channel"), "baz");
|
||||
EXPECT_EQ(*proxy_match("ftp://example.org"), "other");
|
||||
|
||||
Context::instance().proxy_servers = { { "http", "foo" },
|
||||
{ "https", "bar" },
|
||||
{ "https://example.net", "foobar" },
|
||||
{ "all://example.net", "baz" } };
|
||||
|
||||
EXPECT_FALSE(proxy_match("ftp://example.org").has_value());
|
||||
|
||||
Context::instance().proxy_servers = {};
|
||||
|
||||
EXPECT_FALSE(proxy_match("http://example.com/channel").has_value());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -316,6 +316,7 @@ PYBIND11_MODULE(bindings, m)
|
|||
.def_readwrite("always_yes", &Context::always_yes)
|
||||
.def_readwrite("dry_run", &Context::dry_run)
|
||||
.def_readwrite("ssl_verify", &Context::ssl_verify)
|
||||
.def_readwrite("proxy_servers", &Context::proxy_servers)
|
||||
.def_readwrite("max_retries", &Context::max_retries)
|
||||
.def_readwrite("retry_timeout", &Context::retry_timeout)
|
||||
.def_readwrite("retry_backoff", &Context::retry_backoff)
|
||||
|
|
|
|||
|
|
@ -213,6 +213,8 @@ def init_api_context(use_mamba_experimental=False):
|
|||
api_ctx.always_yes = context.always_yes
|
||||
api_ctx.channels = context.channels
|
||||
api_ctx.platform = context.subdir
|
||||
# Conda uses a frozendict here
|
||||
api_ctx.proxy_servers = dict(context.proxy_servers)
|
||||
|
||||
if "MAMBA_EXTRACT_THREADS" in os.environ:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ dependencies:
|
|||
- cmake
|
||||
- ninja
|
||||
- nlohmann_json
|
||||
- mitmproxy
|
||||
- libsolv >=0.7.18
|
||||
- libarchive
|
||||
- libsodium
|
||||
|
|
@ -18,6 +19,7 @@ dependencies:
|
|||
- termcolor-cpp
|
||||
- cli11 >=2.2
|
||||
- pytest
|
||||
- pytest-asyncio
|
||||
- pytest-lazy-fixture
|
||||
- pytest-xprocess
|
||||
- pyyaml
|
||||
|
|
|
|||
|
|
@ -0,0 +1,30 @@
|
|||
"""
|
||||
mitmproxy connection dumper plugin
|
||||
|
||||
This script shouldn't be run or imported directly. Instead, it should be passed to mitmproxy as a script (-s).
|
||||
It will then dump all request urls in the file specified by the outfile option.
|
||||
|
||||
We use this script instead of letting mitmdump do the dumping, because we only care about the urls, while mitmdump
|
||||
also dumps all message content.
|
||||
"""
|
||||
|
||||
from mitmproxy import ctx
|
||||
from mitmproxy.addonmanager import Loader
|
||||
from mitmproxy.http import HTTPFlow
|
||||
|
||||
|
||||
class DumpAddon:
|
||||
def load(self, loader: Loader):
|
||||
loader.add_option(
|
||||
name="outfile",
|
||||
typespec=str,
|
||||
default="",
|
||||
help="Path for the file in which to dump the requests",
|
||||
)
|
||||
|
||||
def request(self, flow: HTTPFlow):
|
||||
with open(ctx.options.outfile, "a+") as f:
|
||||
f.write(flow.request.url + "\n")
|
||||
|
||||
|
||||
addons = [DumpAddon()]
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from subprocess import TimeoutExpired
|
||||
|
||||
from .helpers import *
|
||||
|
||||
|
||||
class TestProxy:
|
||||
|
||||
current_root_prefix = os.environ["MAMBA_ROOT_PREFIX"]
|
||||
current_prefix = os.environ["CONDA_PREFIX"]
|
||||
|
||||
env_name = random_string()
|
||||
root_prefix = os.path.expanduser(os.path.join("~", "tmproot" + random_string()))
|
||||
prefix = os.path.join(root_prefix, "envs", env_name)
|
||||
|
||||
mitm_exe = shutil.which("mitmdump")
|
||||
mitm_confdir = os.path.join(root_prefix, "mitmproxy")
|
||||
mitm_dump_path = os.path.join(root_prefix, "dump.json")
|
||||
|
||||
proxy_process = None
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
os.environ["MAMBA_ROOT_PREFIX"] = TestProxy.root_prefix
|
||||
os.environ["CONDA_PREFIX"] = TestProxy.prefix
|
||||
|
||||
def setup_method(self):
|
||||
create("-n", TestProxy.env_name, "--offline", no_dry_run=True)
|
||||
|
||||
@classmethod
|
||||
def teardown_class(cls):
|
||||
os.environ["MAMBA_ROOT_PREFIX"] = TestProxy.current_root_prefix
|
||||
os.environ["CONDA_PREFIX"] = TestProxy.current_prefix
|
||||
|
||||
def teardown_method(self):
|
||||
shutil.rmtree(TestProxy.root_prefix)
|
||||
|
||||
def start_proxy(self, port, options=[]):
|
||||
assert self.proxy_process is None
|
||||
script = Path(__file__).parent / "dump_proxy_connections.py"
|
||||
self.proxy_process = subprocess.Popen(
|
||||
[
|
||||
TestProxy.mitm_exe,
|
||||
"--listen-port",
|
||||
str(port),
|
||||
"--scripts",
|
||||
script,
|
||||
"--set",
|
||||
f"outfile={TestProxy.mitm_dump_path}",
|
||||
"--set",
|
||||
f"confdir={TestProxy.mitm_confdir}",
|
||||
*options,
|
||||
]
|
||||
)
|
||||
|
||||
# Wait until mitmproxy has generated its certificate or some tests might fail
|
||||
while not (Path(TestProxy.mitm_confdir) / "mitmproxy-ca-cert.pem").exists():
|
||||
time.sleep(1)
|
||||
|
||||
def stop_proxy(self):
|
||||
self.proxy_process.terminate()
|
||||
try:
|
||||
self.proxy_process.wait(3)
|
||||
except TimeoutExpired:
|
||||
self.proxy_process.kill()
|
||||
self.proxy_process = None
|
||||
|
||||
@pytest.mark.parametrize("with_auth", (True, False))
|
||||
@pytest.mark.parametrize("ssl_verify", (True, False))
|
||||
def test_install(self, unused_tcp_port, with_auth, ssl_verify):
|
||||
"""
|
||||
This test makes sure micromamba follows the proxy settings in .condarc
|
||||
|
||||
It starts mitmproxy with the `dump_proxy_connections.py` script, which dumps all requested urls in a text file.
|
||||
After that micromamba is used to install a package, while pointing it to that mitmproxy instance. Once
|
||||
micromamba finished the proxy server is stopped and the urls micromamba requested are compared to the urls
|
||||
mitmproxy intercepted, making sure that all the requests went through the proxy.
|
||||
"""
|
||||
|
||||
if with_auth:
|
||||
proxy_options = ["--proxyauth", "foo:bar"]
|
||||
proxy_url = "http://foo:bar@localhost:{}".format(unused_tcp_port)
|
||||
else:
|
||||
proxy_options = []
|
||||
proxy_url = "http://localhost:{}".format(unused_tcp_port)
|
||||
|
||||
self.start_proxy(unused_tcp_port, proxy_options)
|
||||
|
||||
cmd = ["xtensor"]
|
||||
f_name = random_string() + ".yaml"
|
||||
rc_file = os.path.join(TestProxy.prefix, f_name)
|
||||
|
||||
if ssl_verify:
|
||||
verify_string = os.path.abspath(
|
||||
os.path.join(TestProxy.mitm_confdir, "mitmproxy-ca-cert.pem")
|
||||
)
|
||||
else:
|
||||
verify_string = "false"
|
||||
|
||||
file_content = [
|
||||
"proxy_servers:",
|
||||
" http: {}".format(proxy_url),
|
||||
" https: {}".format(proxy_url),
|
||||
"ssl_verify: {}".format(verify_string),
|
||||
]
|
||||
with open(rc_file, "w") as f:
|
||||
f.write("\n".join(file_content))
|
||||
|
||||
cmd += ["--rc-file", rc_file]
|
||||
|
||||
if os.name == "nt":
|
||||
# The certificates generated by mitmproxy don't support revocation.
|
||||
# The schannel backend curl uses on Windows fails revocation check if revocation isn't supported. Other
|
||||
# backends succeed revocation check in that case.
|
||||
cmd += ["--ssl-no-revoke"]
|
||||
|
||||
res = install(*cmd, "--json", no_rc=False)
|
||||
|
||||
self.stop_proxy()
|
||||
|
||||
with open(TestProxy.mitm_dump_path, "r") as f:
|
||||
proxied_requests = f.read().splitlines()
|
||||
|
||||
for fetch in res["actions"]["FETCH"]:
|
||||
assert fetch["url"] in proxied_requests
|
||||
Loading…
Reference in New Issue