Handle `.tar.gz` in pkg url (#3640)

2024-12-06 08:55:03 +01:00 · 2024-12-06 08:55:03 +01:00 · 6141540b94
parent 8ea3103d3c
commit 6141540b94
6 changed files with 504 additions and 23 deletions
--- a/libmamba/include/mamba/specs/archive.hpp
+++ b/libmamba/include/mamba/specs/archive.hpp
@ -17,7 +17,8 @@ namespace mamba::specs
 {
    inline static constexpr auto ARCHIVE_EXTENSIONS = std::array{ std::string_view(".tar.bz2"),
                                                                  std::string_view(".conda"),
-                                                                  std::string_view(".whl") };
+                                                                  std::string_view(".whl"),
+                                                                  std::string_view(".tar.gz") };

    /** Detect if the package path has one of the known archive extension. */
    template <typename Str, std::enable_if_t<std::is_convertible_v<Str, std::string_view>, bool> = true>
--- a/libmamba/include/mamba/specs/package_info.hpp
+++ b/libmamba/include/mamba/specs/package_info.hpp
@ -24,6 +24,7 @@ namespace mamba::specs
        Unknown,
        Conda,
        Wheel,
+        TarGz,
    };

    class PackageInfo
--- a/libmamba/src/specs/package_info.cpp
+++ b/libmamba/src/specs/package_info.cpp
@ -30,6 +30,10 @@ namespace mamba::specs
            {
                return PackageType::Wheel;
            }
+            else if (util::ends_with(spec, ".tar.gz"))
+            {
+                return PackageType::TarGz;
+            }
            return PackageType::Conda;
        }

@ -40,7 +44,6 @@ namespace mamba::specs
                return make_unexpected_parse("Missing filename extension.");
            }

-
            auto out = PackageInfo();
            // TODO decide on the best way to group filename/channel/subdir/package_url all at once
            out.package_url = util::path_or_url_to_url(spec);
@ -58,37 +61,119 @@ namespace mamba::specs
            out.filename = url.package();
            url.clear_package();

+            // The filename format depends on the package_type:
            out.package_type = parse_extension(spec);
+            // PackageType::Conda (.tar.bz2 or .conda):
+            // {pkg name}-{version}-{build string}.{tar.bz2, conda}
            if (out.package_type == PackageType::Conda)
            {
                out.platform = url.platform_name();
                url.clear_platform();
                out.channel = util::rstrip(url.str(), '/');
+
+                // Note that we use `rsplit...` instead of `split...`
+                // because the package name may contain '-'.
+                // Build string
+                auto [head, tail] = util::rsplit_once(strip_archive_extension(out.filename), '-');
+                out.build_string = tail;
+                if (!head.has_value())
+                {
+                    return make_unexpected_parse(
+                        fmt::format(R"(Missing name and version in filename "{}".)", out.filename)
+                    );
+                }
+
+                // Version
+                std::tie(head, tail) = util::rsplit_once(head.value(), '-');
+                out.version = tail;
+                if (!head.has_value())
+                {
+                    return make_unexpected_parse(
+                        fmt::format(R"(Missing name in filename "{}".)", out.filename)
+                    );
+                }
+
+                // Name
+                out.name = head.value();  // There may be '-' in the name
            }
-
-            // Build string
-            auto [head, tail] = util::rsplit_once(strip_archive_extension(out.filename), '-');
-            out.build_string = tail;
-
-            if (!head.has_value())
+            // PackageType::Wheel (.whl):
+            // {pkg name}-{version}-{build tag (optional)}-{python tag}-{abi tag}-{platform tag}.whl
+            // cf.
+            // https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/
+            else if (out.package_type == PackageType::Wheel)
            {
-                return make_unexpected_parse(
-                    fmt::format(R"(Missing name and version in filename "{}".)", out.filename)
-                );
-            }
+                // Platform tag
+                auto [head, tail] = util::rsplit_once(strip_archive_extension(out.filename), '-');
+                if (!head.has_value())
+                {
+                    return make_unexpected_parse(
+                        fmt::format(R"(Missing tags in filename "{}".)", out.filename)
+                    );
+                }
+                // Abi tag
+                std::tie(head, tail) = util::rsplit_once(head.value(), '-');
+                if (!head.has_value())
+                {
+                    return make_unexpected_parse(
+                        fmt::format(R"(Missing tags in filename "{}".)", out.filename)
+                    );
+                }
+                // Python tag
+                std::tie(head, tail) = util::rsplit_once(head.value(), '-');
+                if (!head.has_value())
+                {
+                    return make_unexpected_parse(
+                        fmt::format(R"(Missing tags in filename "{}".)", out.filename)
+                    );
+                }
+                // Build tag or version
+                std::tie(head, tail) = util::rsplit_once(head.value(), '-');
+                if (!head.has_value())
+                {
+                    return make_unexpected_parse(
+                        fmt::format(R"(Missing tags in filename "{}".)", out.filename)
+                    );
+                }
+                if (util::contains(tail, '.'))
+                {
+                    // The tail is the version
+                    out.version = tail;
+                    // The head is the name
+                    out.name = head.value();  // There may be '-' in the name
+                }
+                else
+                {
+                    // The previous tail is the optional build tag
+                    std::tie(head, tail) = util::rsplit_once(head.value(), '-');
+                    // The tail is the version
+                    out.version = tail;
+                    if (!head.has_value())
+                    {
+                        return make_unexpected_parse(
+                            fmt::format(R"(Missing name in filename "{}".)", out.filename)
+                        );
+                    }

-            // Version
-            std::tie(head, tail) = util::rsplit_once(head.value(), '-');
-            out.version = tail;
-            if (!head.has_value())
+                    // Name
+                    out.name = head.value();  // There may be '-' in the name
+                }
+            }
+            // PackageType::TarGz (.tar.gz): {pkg name}-{version}.tar.gz
+            else if (out.package_type == PackageType::TarGz)
            {
-                return make_unexpected_parse(
-                    fmt::format(R"(Missing name in filename "{}".)", out.filename)
-                );
-            }
+                // Version
+                auto [head, tail] = util::rsplit_once(strip_archive_extension(out.filename), '-');
+                out.version = tail;
+                if (!head.has_value())
+                {
+                    return make_unexpected_parse(
+                        fmt::format(R"(Missing name in filename "{}".)", out.filename)
+                    );
+                }

-            // Name
-            out.name = head.value();  // There may be '-' in the name
+                // Name
+                out.name = head.value();  // There may be '-' in the name
+            }

            return out;
        }
--- a/libmambapy/tests/test_specs.py
+++ b/libmambapy/tests/test_specs.py
@ -24,7 +24,7 @@ def test_ParseError():


 def test_archive_extension():
-    assert libmambapy.specs.archive_extensions() == [".tar.bz2", ".conda", ".whl"]
+    assert libmambapy.specs.archive_extensions() == [".tar.bz2", ".conda", ".whl", ".tar.gz"]

    assert libmambapy.specs.has_archive_extension("pkg.conda")
    assert not libmambapy.specs.has_archive_extension("conda.pkg")
--- a/micromamba/tests/test-env-pip-lock.yaml
+++ b/micromamba/tests/test-env-pip-lock.yaml
@ -0,0 +1,349 @@
+# This lock file was generated by conda-lock (https://github.com/conda/conda-lock). DO NOT EDIT!
+#
+# A "lock file" contains a concrete list of package versions (with checksums) to be installed. Unlike
+# e.g. `conda env create`, the resulting environment will not change as new package versions become
+# available, unless you explicitly update the lock file.
+#
+# Install this environment as "YOURENV" with:
+#     conda-lock install -n YOURENV conda-lock.yml
+# To update a single package to the latest version compatible with the version constraints in the source:
+#     conda-lock lock  --lockfile conda-lock.yml --update PACKAGE
+# To re-solve the entire environment, e.g. after changing a version constraint in the source file:
+#     conda-lock -f env.yml --lockfile conda-lock.yml
+version: 1
+metadata:
+  content_hash:
+    linux-64: 9b9e345bf61ec8a8117b83be4a1e3fe06b653f27a8c342d3792cac0d84182572
+  channels:
+  - url: conda-forge
+    used_env_vars: []
+  - url: bioconda
+    used_env_vars: []
+  platforms:
+  - linux-64
+  sources:
+  - env.yml
+package:
+- name: _libgcc_mutex
+  version: '0.1'
+  manager: conda
+  platform: linux-64
+  dependencies: {}
+  url: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2
+  hash:
+    md5: d7c89558ba9fa0495403155b64376d81
+    sha256: fe51de6107f9edc7aa4f786a70f4a883943bc9d39b3bb7307c04c41410990726
+  category: main
+  optional: false
+- name: _openmp_mutex
+  version: '4.5'
+  manager: conda
+  platform: linux-64
+  dependencies:
+    _libgcc_mutex: '0.1'
+    libgomp: '>=7.5.0'
+  url: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2
+  hash:
+    md5: 73aaf86a425cc6e73fcf236a5a46396d
+    sha256: fbe2c5e56a653bebb982eda4876a9178aedfc2b545f25d0ce9c4c0b508253d22
+  category: main
+  optional: false
+- name: bzip2
+  version: 1.0.8
+  manager: conda
+  platform: linux-64
+  dependencies:
+    __glibc: '>=2.17,<3.0.a0'
+    libgcc-ng: '>=12'
+  url: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda
+  hash:
+    md5: 62ee74e96c5ebb0af99386de58cf9553
+    sha256: 5ced96500d945fb286c9c838e54fa759aa04a7129c59800f0846b4335cee770d
+  category: main
+  optional: false
+- name: ca-certificates
+  version: 2024.8.30
+  manager: conda
+  platform: linux-64
+  dependencies: {}
+  url: https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.8.30-hbcca054_0.conda
+  hash:
+    md5: c27d1c142233b5bc9ca570c6e2e0c244
+    sha256: afee721baa6d988e27fef1832f68d6f32ac8cc99cdf6015732224c2841a09cea
+  category: main
+  optional: false
+- name: ld_impl_linux-64
+  version: '2.43'
+  manager: conda
+  platform: linux-64
+  dependencies:
+    __glibc: '>=2.17,<3.0.a0'
+  url: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.43-h712a8e2_2.conda
+  hash:
+    md5: 048b02e3962f066da18efe3a21b77672
+    sha256: 7c91cea91b13f4314d125d1bedb9d03a29ebbd5080ccdea70260363424646dbe
+  category: main
+  optional: false
+- name: libexpat
+  version: 2.6.4
+  manager: conda
+  platform: linux-64
+  dependencies:
+    __glibc: '>=2.17,<3.0.a0'
+    libgcc: '>=13'
+  url: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.4-h5888daf_0.conda
+  hash:
+    md5: db833e03127376d461e1e13e76f09b6c
+    sha256: 56541b98447b58e52d824bd59d6382d609e11de1f8adf20b23143e353d2b8d26
+  category: main
+  optional: false
+- name: libffi
+  version: 3.4.2
+  manager: conda
+  platform: linux-64
+  dependencies:
+    libgcc-ng: '>=9.4.0'
+  url: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2
+  hash:
+    md5: d645c6d2ac96843a2bfaccd2d62b3ac3
+    sha256: ab6e9856c21709b7b517e940ae7028ae0737546122f83c2aa5d692860c3b149e
+  category: main
+  optional: false
+- name: libgcc
+  version: 14.2.0
+  manager: conda
+  platform: linux-64
+  dependencies:
+    _libgcc_mutex: '0.1'
+    _openmp_mutex: '>=4.5'
+  url: https://conda.anaconda.org/conda-forge/linux-64/libgcc-14.2.0-h77fa898_1.conda
+  hash:
+    md5: 3cb76c3f10d3bc7f1105b2fc9db984df
+    sha256: 53eb8a79365e58849e7b1a068d31f4f9e718dc938d6f2c03e960345739a03569
+  category: main
+  optional: false
+- name: libgcc-ng
+  version: 14.2.0
+  manager: conda
+  platform: linux-64
+  dependencies:
+    libgcc: 14.2.0
+  url: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-14.2.0-h69a702a_1.conda
+  hash:
+    md5: e39480b9ca41323497b05492a63bc35b
+    sha256: 3a76969c80e9af8b6e7a55090088bc41da4cffcde9e2c71b17f44d37b7cb87f7
+  category: main
+  optional: false
+- name: libgomp
+  version: 14.2.0
+  manager: conda
+  platform: linux-64
+  dependencies:
+    _libgcc_mutex: '0.1'
+  url: https://conda.anaconda.org/conda-forge/linux-64/libgomp-14.2.0-h77fa898_1.conda
+  hash:
+    md5: cc3573974587f12dda90d96e3e55a702
+    sha256: 1911c29975ec99b6b906904040c855772ccb265a1c79d5d75c8ceec4ed89cd63
+  category: main
+  optional: false
+- name: libmpdec
+  version: 4.0.0
+  manager: conda
+  platform: linux-64
+  dependencies:
+    __glibc: '>=2.17,<3.0.a0'
+    libgcc-ng: '>=12'
+  url: https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-h4bc722e_0.conda
+  hash:
+    md5: aeb98fdeb2e8f25d43ef71fbacbeec80
+    sha256: d02d1d3304ecaf5c728e515eb7416517a0b118200cd5eacbe829c432d1664070
+  category: main
+  optional: false
+- name: libsqlite
+  version: 3.47.0
+  manager: conda
+  platform: linux-64
+  dependencies:
+    __glibc: '>=2.17,<3.0.a0'
+    libgcc: '>=13'
+    libzlib: '>=1.3.1,<2.0a0'
+  url: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.47.0-hadc24fc_1.conda
+  hash:
+    md5: b6f02b52a174e612e89548f4663ce56a
+    sha256: 8a9aadf996a2399f65b679c6e7f29139d5059f699c63e6d7b50e20db10c00508
+  category: main
+  optional: false
+- name: libuuid
+  version: 2.38.1
+  manager: conda
+  platform: linux-64
+  dependencies:
+    libgcc-ng: '>=12'
+  url: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda
+  hash:
+    md5: 40b61aab5c7ba9ff276c41cfffe6b80b
+    sha256: 787eb542f055a2b3de553614b25f09eefb0a0931b0c87dbcce6efdfd92f04f18
+  category: main
+  optional: false
+- name: libzlib
+  version: 1.3.1
+  manager: conda
+  platform: linux-64
+  dependencies:
+    __glibc: '>=2.17,<3.0.a0'
+    libgcc: '>=13'
+  url: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda
+  hash:
+    md5: edb0dca6bc32e4f4789199455a1dbeb8
+    sha256: d4bfe88d7cb447768e31650f06257995601f89076080e76df55e3112d4e47dc4
+  category: main
+  optional: false
+- name: ncurses
+  version: '6.5'
+  manager: conda
+  platform: linux-64
+  dependencies:
+    __glibc: '>=2.17,<3.0.a0'
+    libgcc-ng: '>=12'
+  url: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-he02047a_1.conda
+  hash:
+    md5: 70caf8bb6cf39a0b6b7efc885f51c0fe
+    sha256: 6a1d5d8634c1a07913f1c525db6455918cbc589d745fac46d9d6e30340c8731a
+  category: main
+  optional: false
+- name: openssl
+  version: 3.3.2
+  manager: conda
+  platform: linux-64
+  dependencies:
+    __glibc: '>=2.17,<3.0.a0'
+    ca-certificates: ''
+    libgcc: '>=13'
+  url: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.3.2-hb9d3cd8_0.conda
+  hash:
+    md5: 4d638782050ab6faa27275bed57e9b4e
+    sha256: cee91036686419f6dd6086902acf7142b4916e1c4ba042e9ca23e151da012b6d
+  category: main
+  optional: false
+- name: pip
+  version: 24.3.1
+  manager: conda
+  platform: linux-64
+  dependencies:
+    python: '>=3.13.0a0'
+  url: https://conda.anaconda.org/conda-forge/noarch/pip-24.3.1-pyh145f28c_0.conda
+  hash:
+    md5: ca3afe2d7b893a8c8cdf489d30a2b1a3
+    sha256: fc305cfe1ad0d51c61dd42a33cf27e03a075992fd0070c173d7cad86c1a48f13
+  category: main
+  optional: false
+- name: python
+  version: 3.13.0
+  manager: conda
+  platform: linux-64
+  dependencies:
+    __glibc: '>=2.17,<3.0.a0'
+    bzip2: '>=1.0.8,<2.0a0'
+    ld_impl_linux-64: '>=2.36.1'
+    libexpat: '>=2.6.3,<3.0a0'
+    libffi: '>=3.4,<4.0a0'
+    libgcc: '>=13'
+    libmpdec: '>=4.0.0,<5.0a0'
+    libsqlite: '>=3.46.1,<4.0a0'
+    libuuid: '>=2.38.1,<3.0a0'
+    libzlib: '>=1.3.1,<2.0a0'
+    ncurses: '>=6.5,<7.0a0'
+    openssl: '>=3.3.2,<4.0a0'
+    python_abi: 3.13.*
+    readline: '>=8.2,<9.0a0'
+    tk: '>=8.6.13,<8.7.0a0'
+    tzdata: ''
+    xz: '>=5.2.6,<6.0a0'
+    pip: ''
+  url: https://conda.anaconda.org/conda-forge/linux-64/python-3.13.0-h9ebbce0_100_cp313.conda
+  hash:
+    md5: 08e9aef080f33daeb192b2ddc7e4721f
+    sha256: 6ab5179679f0909db828d8316f3b8b379014a82404807310fe7df5a6cf303646
+  category: main
+  optional: false
+- name: python_abi
+  version: '3.13'
+  manager: conda
+  platform: linux-64
+  dependencies: {}
+  url: https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.13-5_cp313.conda
+  hash:
+    md5: 381bbd2a92c863f640a55b6ff3c35161
+    sha256: 438225b241c5f9bddae6f0178a97f5870a89ecf927dfca54753e689907331442
+  category: main
+  optional: false
+- name: readline
+  version: '8.2'
+  manager: conda
+  platform: linux-64
+  dependencies:
+    libgcc-ng: '>=12'
+    ncurses: '>=6.3,<7.0a0'
+  url: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda
+  hash:
+    md5: 47d31b792659ce70f470b5c82fdfb7a4
+    sha256: 5435cf39d039387fbdc977b0a762357ea909a7694d9528ab40f005e9208744d7
+  category: main
+  optional: false
+- name: tk
+  version: 8.6.13
+  manager: conda
+  platform: linux-64
+  dependencies:
+    libgcc-ng: '>=12'
+    libzlib: '>=1.2.13,<2.0.0a0'
+  url: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda
+  hash:
+    md5: d453b98d9c83e71da0741bb0ff4d76bc
+    sha256: e0569c9caa68bf476bead1bed3d79650bb080b532c64a4af7d8ca286c08dea4e
+  category: main
+  optional: false
+- name: tzdata
+  version: 2024b
+  manager: conda
+  platform: linux-64
+  dependencies: {}
+  url: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024b-hc8b5060_0.conda
+  hash:
+    md5: 8ac3367aafb1cc0a068483c580af8015
+    sha256: 4fde5c3008bf5d2db82f2b50204464314cc3c91c1d953652f7bd01d9e52aefdf
+  category: main
+  optional: false
+- name: xz
+  version: 5.2.6
+  manager: conda
+  platform: linux-64
+  dependencies:
+    libgcc-ng: '>=12'
+  url: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2
+  hash:
+    md5: 2161070d867d1b1204ea749c8eec4ef0
+    sha256: 03a6d28ded42af8a347345f82f3eebdd6807a08526d47899a42d62d319609162
+  category: main
+  optional: false
+- name: checkm
+  version: '0.4'
+  manager: pip
+  platform: linux-64
+  dependencies: {}
+  url: https://files.pythonhosted.org/packages/e4/2f/b6ad927d467451a1b5872cce8e7204ec25d2a6cde8077cb28003ed35787d/Checkm-0.4.tar.gz
+  hash:
+    sha256: 2d09d77c85d5b4158ec699c5b0a33b3b1168fc1aba146ed54a634f68ddca1fa6
+  category: main
+  optional: false
+- name: starlette
+  version: '0.17.1'
+  manager: pip
+  platform: linux-64
+  dependencies:
+    anyio: '>=3.0.0,<4'
+  url: https://files.pythonhosted.org/packages/32/57/e9c68acc2845ee4ca66202d19856f6a3581cab2a885d25d490103270ffa2/starlette-0.17.1-py3-none-any.whl
+  hash:
+    sha256: 26a18cbda5e6b651c964c12c88b36d9898481cd428ed6e063f5f29c418f73050
+  category: main
+  optional: false
--- a/micromamba/tests/test_create.py
+++ b/micromamba/tests/test_create.py
@ -24,6 +24,7 @@ env_files = [
 ]

 lockfile_path: Path = __this_dir__ / "test_env-lock.yaml"
+pip_lockfile_path: Path = __this_dir__ / "test-env-pip-lock.yaml"


 def check_create_result(res, root_prefix, target_prefix):
@ -114,6 +115,50 @@ def test_lockfile(tmp_home, tmp_root_prefix, tmp_path):
    assert any(package["name"] == "zlib" and package["version"] == "1.2.11" for package in packages)


+@pytest.mark.skipif(
+    platform.system() != "Linux",
+    reason="Test only available on Linux (cf. `test-env-pip-lock.yaml`)",
+)
+@pytest.mark.parametrize("shared_pkgs_dirs", [True], indirect=True)
+def test_lockfile_with_pip(tmp_home, tmp_root_prefix, tmp_path):
+    env_prefix = tmp_path / "myenv"
+    spec_file = tmp_path / "pip-env-lock.yaml"
+
+    shutil.copyfile(pip_lockfile_path, spec_file)
+
+    res = helpers.create("-p", env_prefix, "-f", spec_file, "--json")
+    assert res["success"]
+
+    packages = helpers.umamba_list("-p", env_prefix, "--json")
+
+    # Test pkg url ending with `.tar.gz`
+    assert any(
+        package["name"] == "Checkm" and package["version"] == "0.4" and package["channel"] == "pypi"
+        for package in packages
+    )
+    # Test pkg url ending with `.whl`
+    assert any(
+        package["name"] == "starlette"
+        and package["version"] == "0.17.1"
+        and package["channel"] == "pypi"
+        for package in packages
+    )
+    # Test pkg url ending with `.conda`
+    assert any(
+        package["name"] == "bzip2"
+        and package["version"] == "1.0.8"
+        and package["channel"] == "conda-forge"
+        for package in packages
+    )
+    # Test pkg url ending with `.tar.bz2`
+    assert any(
+        package["name"] == "xz"
+        and package["version"] == "5.2.6"
+        and package["channel"] == "conda-forge"
+        for package in packages
+    )
+
+
@pytest.mark.parametrize("shared_pkgs_dirs", [True], indirect=True)
 def test_lockfile_online(tmp_home, tmp_root_prefix, tmp_path):
    env_prefix = tmp_path / "myenv"