ci: Rerun pytest tests on `main` in case of failures (#3769)

Signed-off-by: Julien Jerphanion <git@jjerphan.xyz>
Co-authored-by: Johan Mabille <johan.mabille@gmail.com>
Co-authored-by: Hind Montassif <hind.montassif@gmail.com>

.github/workflows/unix_impl.yml

@@ -107,7 +107,10 @@ jobs:
           python -m pip install --no-deps --no-build-isolation ./libmambapy
       - name: Run libmamba Python bindings tests
         run: |
-          python -m pytest libmambapy/tests/ ${{ runner.debug == 'true' && '-v' || '--exitfirst' }}
+          # Only rerun flaky tests on the `main` branch
+          python -m pytest libmambapy/tests/ \
+              ${{ runner.debug == 'true' && '-v --capture=tee-sys' || '--exitfirst' }} \
+              ${{ github.ref == 'refs/heads/main' && '--reruns 3' || '' }}
 
   mamba_integration_tests_unix:
     name: mamba integration tests
@@ -162,8 +165,10 @@ jobs:
         run: |
           export TEST_MAMBA_EXE=$(pwd)/build/micromamba/mamba
           unset CONDARC  # Interferes with tests
+          # Only rerun flaky tests on the `main` branch
           python -m pytest micromamba/tests/ \
-            ${{ runner.debug == 'true' && '-v --capture=tee-sys' || '--exitfirst' }}
+              ${{ runner.debug == 'true' && '-v --capture=tee-sys' || '--exitfirst' }} \
+              ${{ github.ref == 'refs/heads/main' && '--reruns 3' || '' }}
 
   verify_pkg_tests:
     name: mamba-content-trust tests

.github/workflows/windows_impl.yml

@@ -116,7 +116,10 @@ jobs:
           python -m pip install --no-deps --no-build-isolation ./libmambapy
       - name: Run libmamba Python bindings tests
         run: |
-          python -m pytest libmambapy/tests/ ${{ runner.debug == 'true' && '-v' || '--exitfirst' }}
+          # Only rerun flaky tests on the `main` branch
+          python -m pytest libmambapy/tests/ \
+              ${{ runner.debug == 'true' && '-v --capture=tee-sys' || '--exitfirst' }} \
+              ${{ github.ref == 'refs/heads/main' && '--reruns 3' || '' }}
 
   mamba_integration_tests_win:
     name: mamba integration tests
@@ -149,5 +152,7 @@ jobs:
           $env:TEST_MAMBA_EXE = Join-Path -Path $pwd -ChildPath 'local\bin\mamba.exe'
           $env:MAMBA_TEST_SHELL_TYPE='powershell'
           Remove-Item -Path "env:CONDARC"
-          python -m pytest micromamba/tests/ `
-            ${{ runner.debug == 'true' && '-v --capture=tee-sys' || '--exitfirst' }}
+          # Only rerun flaky tests on the `main` branch
+          python -m pytest micromamba/tests/ \
+              ${{ runner.debug == 'true' && '-v --capture=tee-sys' || '--exitfirst' }} \
+              ${{ github.ref == 'refs/heads/main' && '--reruns 3' || '' }}

dev/environment-dev.yml

@@ -31,6 +31,7 @@ dependencies:
   - pytest-asyncio
   - pytest-timeout
   - pytest-xprocess
+  - pytest-rerunfailures
   - memory_profiler
   - requests
   - sel(win): pywin32