Dev workflow (#2948)

* Add dev-extra tools

* Refactor CMake Presets

* Add taskfile

* Rename Taskfile

* Add .gitignore entry

.gitignore

@@ -1,6 +1,12 @@
 # Git files
 *.orig
 
+# User Taskfile.yml
+Taskfile.yml
+Taskfile.yaml
+taskfile.yml
+taskfile.yaml
+
 # Mamba generated files
 include/mamba/core/version.hpp
 

CMakePresets.json

@@ -1,10 +1,4 @@
 {
-  "buildPresets": [
-    {
-      "configurePreset": "mamba-shared-debug",
-      "name": "mamba-shared-debug"
-    }
-  ],
   "configurePresets": [
     {
       "binaryDir": "${sourceDir}/build/${presetName}",
@@ -27,10 +21,10 @@
         "CMAKE_RANLIB": "$env{RANLIB}",
         "CMAKE_STRIP": "$env{STRIP}"
       },
-      "description": "Base profile using conda libraries and compilers",
-      "displayName": "Conda Base",
+      "description": "Base profile using conda libraries and compilers on Unix",
+      "displayName": "Conda Unix",
       "hidden": true,
-      "name": "conda-base"
+      "name": "conda-unix"
     },
     {
       "cacheVariables": {
@@ -42,10 +36,10 @@
         "CMAKE_RANLIB": "$env{GCC_RANLIB}"
       },
       "description": "Base profile using conda libraries and GNU compilers",
-      "displayName": "Conda Default",
+      "displayName": "Conda GNU",
       "hidden": true,
       "inherits": [
-        "conda-base"
+        "conda-unix"
       ],
       "name": "conda-gnu"
     },
@@ -60,7 +54,7 @@
       "displayName": "Conda Clang",
       "hidden": true,
       "inherits": [
-        "conda-base"
+        "conda-unix"
       ],
       "name": "conda-llvm"
     },
@@ -71,20 +65,12 @@
         "BUILD_LIBMAMBA_TESTS": "ON",
         "BUILD_MAMBA_PACKAGE": "ON",
         "BUILD_MICROMAMBA": "ON",
-        "BUILD_MICROMAMBA_SERVER": "ON",
-        "CMAKE_CXX_EXTENSIONS": "OFF",
-        "CMAKE_CXX_STANDARD": "17",
-        "CMAKE_CXX_STANDARD_REQUIRED": "ON",
-        "CMAKE_EXPORT_COMPILE_COMMANDS": "ON"
+        "BUILD_MICROMAMBA_SERVER": "ON"
       },
-      "description": "Base profile for development in Mamba",
-      "displayName": "Mamba dev",
+      "description": "Base profile for building everything in Mamba",
+      "displayName": "Mamba All",
       "hidden": true,
-      "name": "mamba-dev",
-      "warnings": {
-        "dev": true,
-        "unusedCli": true
-      }
+      "name": "mamba-all"
     },
     {
       "cacheVariables": {
@@ -108,14 +94,35 @@
       "name": "mamba-debug"
     },
     {
-      "displayName": "Mamba shared debug",
+      "cacheVariables": {
+        "CMAKE_COLOR_DIAGNOSTICS": "ON",
+        "CMAKE_COLOR_MAKEFILE": "ON",
+        "CMAKE_CXX_COMPILER_LAUNCHER": "ccache",
+        "CMAKE_CXX_EXTENSIONS": "OFF",
+        "CMAKE_CXX_STANDARD": "17",
+        "CMAKE_CXX_STANDARD_REQUIRED": "ON",
+        "CMAKE_C_COMPILER_LAUNCHER": "ccache",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": "ON"
+      },
+      "description": "Extra convenience flags used when developping",
+      "displayName": "Mamba Dev",
+      "hidden": true,
+      "name": "mamba-dev",
+      "warnings": {
+        "dev": true,
+        "unusedCli": true
+      }
+    },
+    {
+      "displayName": "Mamba Unix Shared Debug Dev",
       "inherits": [
-        "conda-base",
-        "mamba-dev",
+        "conda-unix",
+        "mamba-all",
         "mamba-shared",
-        "mamba-debug"
+        "mamba-debug",
+        "mamba-dev"
       ],
-      "name": "mamba-shared-debug"
+      "name": "mamba-unix-shared-debug-dev"
     }
   ],
   "version": 4

Taskfile.dist.yml

@@ -0,0 +1,171 @@
+version: '3'
+
+vars:
+  BUILD_DIR: 'build'
+  ENV_DIR: '{{.BUILD_DIR}}/envs'
+  DEV_ENV_DIR: '{{.ENV_DIR}}/dev'
+  DEV_RUN: 'micromamba run --prefix {{.DEV_ENV_DIR}}'
+  TEST_ENV_DIR: '{{.ENV_DIR}}/test'
+  TEST_RUN: 'micromamba run --prefix {{.TEST_ENV_DIR}}'
+  CMAKE_BUILD_DIR: '{{.BUILD_DIR}}/cmake'
+  CMAKE_PRESET: 'mamba-unix-shared-debug-dev'
+  CACHE_DIR: '{{.BUILD_DIR}}/pkgs'
+  CPU_PERCENTAGE: 75
+  CPU_COUNT:
+    sh: >-
+      {{- if eq OS "linux" -}}
+        echo $(($(cat /proc/cpuinfo | grep processor | wc -l) * {{.CPU_PERCENTAGE}} / 100))
+      {{- else -}}
+        echo 1
+      {{- end -}}
+
+tasks:
+  _create-env:
+    run: 'when_changed'
+    internal: true
+    cmds:
+      # Pin some version of Python so that we don' develop witha too recent version
+      - 'micromamba create --yes --prefix "{{.prefix}}" python=3.9'
+      - 'micromamba install --yes --prefix "{{.prefix}}" --file ./dev/environment-dev.yml'
+      - 'micromamba install --yes --prefix "{{.prefix}}" --file ./dev/environment-dev-extra.yml'
+    sources:
+      - './dev/environment-dev.yml'
+      - './dev/environment-dev-extra.yml'
+    generates:
+      - '{{.prefix}}/conda-meta/**/*'
+
+  _copy-env:
+    internal: true
+    cmds:
+      - 'micromamba env export --explicit --prefix "{{.source_prefix}}" > "{{.BUILD_DIR}}/env.tmp.lock"'
+      - defer: 'rm "{{.BUILD_DIR}}/env.tmp.lock"'
+      - 'micromamba create --yes --prefix "{{.prefix}}" --offline --file "{{.BUILD_DIR}}/env.tmp.lock"'
+    sources:
+      - '{{.source_prefix}}/conda-meta/**/*'
+    generates:
+      - '{{.prefix}}/conda-meta/**/*'
+
+  create-dev-env:
+    summary: 'Create a local development mamba environment with all needed dependencies'
+    cmds: [{task: '_create-env', vars: {prefix: '{{.DEV_ENV_DIR}}'}}]
+
+  create-test-env:
+    summary: 'Copy the development environment'
+    deps: [create-dev-env]
+    cmds:
+      - task: '_copy-env'
+        vars: {source_prefix: '{{.DEV_ENV_DIR}}', prefix: '{{.TEST_ENV_DIR}}'}
+
+  _pre-commit:
+    internal: true
+    summary: 'Run pre-commit hooks inside the development environment'
+    deps: [create-dev-env]
+    cmds:
+      - cmd: '{{.DEV_RUN}} pre-commit run {{.args}}'
+        ignore_error: true
+  pre-commit:
+    cmds: [{task: _pre-commit, vars: {args: '{{.CLI_ARGS | default "--all-files"}}'}}]
+
+  _configure:
+    internal: true
+    deps: ['create-dev-env', 'create-test-env']
+    cmds:
+      - >-
+        {{.DEV_RUN}} cmake -B "{{.CMAKE_BUILD_DIR}}"
+        --preset {{.CMAKE_PRESET}}
+        -D CMAKE_INSTALL_PREFIX="{{.TEST_ENV_DIR}}"
+        -D CMAKE_INSTALL_RPATH="{{.PWD}}/{{.TEST_ENV_DIR}}/lib"
+        {{.args}}
+      - 'ln -sf "{{.CMAKE_BUILD_DIR}}/compile_commands.json"'
+    # CMake knows when and how it needs to reconfigure itself, so we set that task to
+    # only run for the first configuration
+    status:
+      - test -f '{{.CMAKE_BUILD_DIR}}/CMakeCache.txt'
+  configure:
+    cmds: [{task: _configure, vars: {args: '{{.CLI_ARGS}}'}}]
+  reconfigure:
+    cmds:
+      - 'rm -rf "{{.CMAKE_BUILD_DIR}}/CMakeCache.txt"'
+      - {task: _configure, vars: {args: '{{.CLI_ARGS}}'}}
+
+  _build:
+    internal: true
+    deps: ['_configure']
+    cmds:
+      - '{{.DEV_RUN}} cmake --build "{{.CMAKE_BUILD_DIR}}" {{.args}}'
+    vars:
+      cpu_count: '{{.cpu_count | default .CPU_COUNT}}'
+      target: '{{.target | default "all"}}'
+      args: '{{.args | default (printf "--parallel %s --target %s" .cpu_count .target)}}'
+    # Again, CMake knows what to rebuild so we always rerun the task
+  build:
+    cmds: [{task: '_build', vars: {args: '{{.CLI_ARGS}}'}}]
+
+  micromamba:
+    deps: [{task: '_build', vars: {target: 'micromamba'}}]
+    cmds:
+      - '"{{.CMAKE_BUILD_DIR}}/micromamba/micromamba" {{.CLI_ARGS}}'
+
+  _test-libmamba:
+    internal: true
+    deps: [{task: _build, vars: {target: 'test_libmamba'}}]
+    dir: '{{.CMAKE_BUILD_DIR}}/libmamba/tests/'
+    cmds:
+      - './test_libmamba {{.args}}'
+  test-libmamba:
+    cmds: [{task: '_test-libmamba', vars: {args: '{{.CLI_ARGS}}'}}]
+
+  _test-micromamba:
+    internal: true
+    deps: [{task: '_build', vars: {target: 'micromamba'}}]
+    env:
+      TEST_MAMBA_EXE:
+        sh: 'realpath {{.CMAKE_BUILD_DIR}}/micromamba/micromamba'
+    cmds:
+      - >-
+        {{.DEV_RUN}} python -m pytest micromamba/tests/
+        --mamba-pkgs-dir="{{.CACHE_DIR}}" {{.args}}
+  test-micromamba:
+    cmds: [{task: _test-micromamba, vars: {args: '{{.CLI_ARGS}}'}}]
+
+  install-cpp:
+    deps: [_build]
+    cmds:
+      - '{{.DEV_RUN}} cmake --install "{{.CMAKE_BUILD_DIR}}" --prefix "{{.TEST_ENV_DIR}}"'
+
+  _test-reposerver:
+    internal: true
+    deps: [{task: '_build', vars: {target: 'micromamba'}}]
+    env:
+      TEST_MAMBA_EXE:
+        sh: 'realpath {{.CMAKE_BUILD_DIR}}/micromamba/micromamba'
+    # Explicitly using this as var since env does not override shell environment
+    vars:
+      GNUPGHOME: '{{.BUILD_DIR}}/gnupg'
+    cmds:
+      - mkdir -p {{.GNUPGHOME}}
+      - defer: 'rm -rf "{{.GNUPGHOME}}"'
+      - '{{.DEV_RUN}} --env GNUPGHOME={{.GNUPGHOME}} ./micromamba/test-server/generate_gpg_keys.sh'
+      - '{{.DEV_RUN}} --env GNUPGHOME={{.GNUPGHOME}} ./micromamba/test-server/testserver.sh {{.args}}'
+  test-reposerver:
+    cmds: [{task: _test-reposerver, vars: {args: '{{.CLI_ARGS}}'}}]
+
+  install-py:
+    deps: [install-cpp]
+    cmds:
+      - >-
+        {{.TEST_RUN}} python -m pip install
+        --no-deps --no-build-isolation --ignore-installed --editable libmambapy/
+
+  stubgen:
+    deps: [install-py]
+    cmds:
+      - '{{.TEST_RUN}} python -m pybind11_stubgen -o "{{.BUILD_DIR}}/stubs" libmambapy.bindings'
+      - cp "{{.BUILD_DIR}}/stubs/libmambapy/bindings-stubs/__init__.pyi" libmambapy/libmambapy/
+      - '{{.DEV_RUN}} pre-commit run --files libmambapy/libmambapy/__init__.pyi'
+
+  clean: 'rm -rf {{.BUILD_DIR}}'
+
+  clean-ci-caches: >-
+    gh api 'repos/{owner}/{repo}/actions/caches' --paginate --jq '.actions_caches[].id'
+    | xargs -I'{}' gh api -X DELETE 'repos/{owner}/{repo}/actions/caches/{}'

dev/environment-dev-extra.yml

@@ -0,0 +1,26 @@
+channels:
+  - conda-forge
+dependencies:
+  # Compiler cache
+  - ccache
+  # C++ LSP tools and related packages
+  - clang-tools
+  - clang
+  - clangxx
+  - lld
+  # C++ Debugging
+  - gdb
+  - valgrind
+  # Python LSP support
+  - python-lsp-server-base
+  - python-lsp-black
+  - black
+  # Interactive Python tools
+  - jupyterlab
+  - ipython
+  # Python Debugging
+  - ipdb
+  # Github CLI tool
+  - gh
+  # Taskfile for running scripts
+  - go-task