From 36a7d48f13a9833b8718b10efd292476d7bc4fb9 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 27 May 2025 12:14:38 -0700 Subject: [PATCH 01/15] Defining the GDAL version from the debian install. RE:#1952 --- docker/Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 52f610697..010450885 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,9 +18,15 @@ FROM mambaorg/micromamba:1.5.0-bookworm-slim ARG PYTHON_VERSION="3.11" COPY --from=build /invest/dist/*.whl /tmp/ + +# Define this to be the GDAL version used to build the InVEST wheel in the debian step. +# Check the debian package repo for the debian release above to confirm the version. +# Debian's GDAL version will undoubtedly be earlier than what's available in conda-forge. +ARG DEBIAN_GDAL_VERSION="3.6" + # The environment.yml file will be built during github actions. COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/environment.yml -RUN micromamba install -y -n base -c conda-forge python==${PYTHON_VERSION} && \ +RUN micromamba install -y -n base -c conda-forge python==${PYTHON_VERSION} gdal=${DEBIAN_GDAL_VERSION} && \ micromamba install -y -n base -c conda-forge -f /tmp/environment.yml && \ micromamba clean --all --yes && \ /opt/conda/bin/python -m pip install /tmp/*.whl && \ From d73143954c9f00bee2ca70d58ce3ddfc4000ab92 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 27 May 2025 12:17:17 -0700 Subject: [PATCH 02/15] Updating base container versions. RE:#1952 --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 010450885..7e19737ea 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 # Build the InVEST wheel in a separate container stage -FROM debian:12.2 AS build +FROM debian:12.11 AS build ARG INVEST_VERSION="main" ARG INVEST_REPO="natcap/invest" RUN apt update && apt install -y python3 python3-dev python3-pip python3-build build-essential git python3.11-venv libgdal-dev @@ -12,7 +12,7 @@ RUN cd / && \ python3 -m build # Create the container for distribution that has runtime dependencies. -FROM mambaorg/micromamba:1.5.0-bookworm-slim +FROM mambaorg/micromamba:2.1.1-debian12-slim # Python version should match the version used in stage 1. # If we update the stage 1 debian version, also update this python version ARG PYTHON_VERSION="3.11" From d69cf77efe3c72d81da7e65725423bf79d1fefdd Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 27 May 2025 13:05:03 -0700 Subject: [PATCH 03/15] Adding the pull_request trigger for future PR builds. --- .github/workflows/build-docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 47be5d862..172880007 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -2,6 +2,7 @@ name: Build containers on: workflow_dispatch: + pull_request: push: paths: - 'docker/*' From ada7e6d10c20bc9413f00356ce2ba4a4846a6727 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 27 May 2025 13:16:22 -0700 Subject: [PATCH 04/15] Correcting which repo we're building against. RE:#1952 --- .github/workflows/build-docker.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 172880007..b6ee7f466 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -56,8 +56,15 @@ jobs: run: | # Replace / (invalid tag character) with . SANITIZED_REF="$(echo ${{github.ref_name}} | sed 's|/|.|g')" + if [[ "${{github.event_name }}" == "pull_request" ]] + then + # If we're in a PR, build the container from the fork's source. + INVEST_REPO="${{ github.event.pull_request.head.repo.full_name }}" + else + INVEST_REPO="${{ github.repository }}" + fi cd docker && docker build \ - --build-arg="INVEST_REPO=${{ github.repository }}" \ + --build-arg="INVEST_REPO=${INVEST_REPO}" \ --build-arg="INVEST_VERSION=${{ github.sha }}" \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:latest \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${{ github.sha }} \ From 4644a0f4738f35d1127102d10d02234990573445 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 27 May 2025 13:22:58 -0700 Subject: [PATCH 05/15] Using the merge commit during a PR. RE:#1952 --- .github/workflows/build-docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index b6ee7f466..3cb5b17d7 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -60,12 +60,14 @@ jobs: then # If we're in a PR, build the container from the fork's source. INVEST_REPO="${{ github.event.pull_request.head.repo.full_name }}" + INVEST_VERSION="${{ github.event.pull_request.merge_commit_sha }}" else INVEST_REPO="${{ github.repository }}" + INVEST_VERSION="${{ github.sha }}" fi cd docker && docker build \ --build-arg="INVEST_REPO=${INVEST_REPO}" \ - --build-arg="INVEST_VERSION=${{ github.sha }}" \ + --build-arg="INVEST_VERSION=${INVEST_VERSION}" \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:latest \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${{ github.sha }} \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${SANITIZED_REF} \ From 1b9e71b79be9d60fb8175e7705f903f1453b0ba4 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 27 May 2025 13:27:39 -0700 Subject: [PATCH 06/15] PR commit is on the upstream repo, not the fork. RE:#1952 --- .github/workflows/build-docker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 3cb5b17d7..ab3984ad4 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -59,7 +59,8 @@ jobs: if [[ "${{github.event_name }}" == "pull_request" ]] then # If we're in a PR, build the container from the fork's source. - INVEST_REPO="${{ github.event.pull_request.head.repo.full_name }}" + #INVEST_REPO="${{ github.event.pull_request.head.repo.full_name }}" + INVEST_REPO="${{ github.repository }}" INVEST_VERSION="${{ github.event.pull_request.merge_commit_sha }}" else INVEST_REPO="${{ github.repository }}" From 589f68ad37c78295b366f0ff431e68fea6969101 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 27 May 2025 13:35:27 -0700 Subject: [PATCH 07/15] Trying to build the container from current state. RE:#1952 --- .github/workflows/build-docker.yml | 4 ++-- docker/Dockerfile | 9 ++------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index ab3984ad4..e8ff2e9f8 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -66,9 +66,9 @@ jobs: INVEST_REPO="${{ github.repository }}" INVEST_VERSION="${{ github.sha }}" fi + #--build-arg="INVEST_REPO=${INVEST_REPO}" \ + #--build-arg="INVEST_VERSION=${INVEST_VERSION}" \ cd docker && docker build \ - --build-arg="INVEST_REPO=${INVEST_REPO}" \ - --build-arg="INVEST_VERSION=${INVEST_VERSION}" \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:latest \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${{ github.sha }} \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${SANITIZED_REF} \ diff --git a/docker/Dockerfile b/docker/Dockerfile index 7e19737ea..2226f3fc5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,14 +2,9 @@ # Build the InVEST wheel in a separate container stage FROM debian:12.11 AS build -ARG INVEST_VERSION="main" -ARG INVEST_REPO="natcap/invest" RUN apt update && apt install -y python3 python3-dev python3-pip python3-build build-essential git python3.11-venv libgdal-dev -RUN cd / && \ - git clone https://github.com/${INVEST_REPO}.git && \ - cd $(basename ${INVEST_REPO}) && \ - git checkout ${INVEST_VERSION} && \ - python3 -m build +ADD .. +RUN python3 -m build # Create the container for distribution that has runtime dependencies. FROM mambaorg/micromamba:2.1.1-debian12-slim From e520c986d66c5d5cd554304e0664ab90e3540029 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 27 May 2025 13:37:04 -0700 Subject: [PATCH 08/15] Correcting adding in the invest source. RE:#1952 --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2226f3fc5..78ed9568e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,8 +3,8 @@ # Build the InVEST wheel in a separate container stage FROM debian:12.11 AS build RUN apt update && apt install -y python3 python3-dev python3-pip python3-build build-essential git python3.11-venv libgdal-dev -ADD .. -RUN python3 -m build +ADD .. /invest +RUN cd /invest && python3 -m build # Create the container for distribution that has runtime dependencies. FROM mambaorg/micromamba:2.1.1-debian12-slim From d379a2d2d0cd1c8f34cb0484408765f4f8a3cd26 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 27 May 2025 14:01:17 -0700 Subject: [PATCH 09/15] Correcting how docker is built to fix missing build context. RE:#1952 --- .github/workflows/build-docker.yml | 2 +- docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index e8ff2e9f8..63e44e0d6 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -68,7 +68,7 @@ jobs: fi #--build-arg="INVEST_REPO=${INVEST_REPO}" \ #--build-arg="INVEST_VERSION=${INVEST_VERSION}" \ - cd docker && docker build \ + docker build -f docker/Dockerfile . \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:latest \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${{ github.sha }} \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${SANITIZED_REF} \ diff --git a/docker/Dockerfile b/docker/Dockerfile index 78ed9568e..2fee84e29 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,7 +3,7 @@ # Build the InVEST wheel in a separate container stage FROM debian:12.11 AS build RUN apt update && apt install -y python3 python3-dev python3-pip python3-build build-essential git python3.11-venv libgdal-dev -ADD .. /invest +ADD . /invest RUN cd /invest && python3 -m build # Create the container for distribution that has runtime dependencies. From 3e78b6c044fe78eb12d083aa7de8a579015a7397 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 27 May 2025 14:02:54 -0700 Subject: [PATCH 10/15] Correcting docker build call, removing old logic. RE:#1952 --- .github/workflows/build-docker.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 63e44e0d6..9d294284d 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -56,19 +56,7 @@ jobs: run: | # Replace / (invalid tag character) with . SANITIZED_REF="$(echo ${{github.ref_name}} | sed 's|/|.|g')" - if [[ "${{github.event_name }}" == "pull_request" ]] - then - # If we're in a PR, build the container from the fork's source. - #INVEST_REPO="${{ github.event.pull_request.head.repo.full_name }}" - INVEST_REPO="${{ github.repository }}" - INVEST_VERSION="${{ github.event.pull_request.merge_commit_sha }}" - else - INVEST_REPO="${{ github.repository }}" - INVEST_VERSION="${{ github.sha }}" - fi - #--build-arg="INVEST_REPO=${INVEST_REPO}" \ - #--build-arg="INVEST_VERSION=${INVEST_VERSION}" \ - docker build -f docker/Dockerfile . \ + docker build -f docker/Dockerfile \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:latest \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${{ github.sha }} \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${SANITIZED_REF} \ From 44e9b5c98612cda943a9cfb91f859b7c372893d9 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 27 May 2025 14:04:21 -0700 Subject: [PATCH 11/15] Correcting path to environment file. RE:#1952 --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2fee84e29..59e6b7974 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,7 +20,7 @@ COPY --from=build /invest/dist/*.whl /tmp/ ARG DEBIAN_GDAL_VERSION="3.6" # The environment.yml file will be built during github actions. -COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/environment.yml +COPY --chown=$MAMBA_USER:$MAMBA_USER docker/environment.yml /tmp/environment.yml RUN micromamba install -y -n base -c conda-forge python==${PYTHON_VERSION} gdal=${DEBIAN_GDAL_VERSION} && \ micromamba install -y -n base -c conda-forge -f /tmp/environment.yml && \ micromamba clean --all --yes && \ From faf8b06b636d3980400ed2f11576c08c67f49933 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 27 May 2025 14:06:55 -0700 Subject: [PATCH 12/15] Cloning full git history for better versioning. RE:#1952 --- .github/workflows/build-docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 9d294284d..7cd078cb9 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -40,6 +40,8 @@ jobs: packages: write steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: login to GitHub Container Registry uses: docker/login-action@v2 From fdf0afdab05a7b425efdd73fbb456a3f8c0aa78c Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 27 May 2025 14:12:57 -0700 Subject: [PATCH 13/15] Clarifying files needed for wheel build. RE:#1952 --- docker/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 59e6b7974..073409e4d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,7 +3,11 @@ # Build the InVEST wheel in a separate container stage FROM debian:12.11 AS build RUN apt update && apt install -y python3 python3-dev python3-pip python3-build build-essential git python3.11-venv libgdal-dev -ADD . /invest + +# only copy in the files needed for building the wheel +ADD src /invest/src +ADD pyproject.toml /invest/pyproject.toml +ADD setup.py /invest/setup.py RUN cd /invest && python3 -m build # Create the container for distribution that has runtime dependencies. From 04c03a59d24118bd1ae84c45e304d926bff99a6a Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 27 May 2025 14:15:16 -0700 Subject: [PATCH 14/15] Adding requirements.txt. RE:#1952 --- docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 073409e4d..7de4b8e43 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,6 +8,7 @@ RUN apt update && apt install -y python3 python3-dev python3-pip python3-build b ADD src /invest/src ADD pyproject.toml /invest/pyproject.toml ADD setup.py /invest/setup.py +ADD requirements.txt /invest/requirements.txt RUN cd /invest && python3 -m build # Create the container for distribution that has runtime dependencies. From 95517eca936e2549a6941a8837890f50c4130a62 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 27 May 2025 14:17:21 -0700 Subject: [PATCH 15/15] Adding the full git repo. RE:#1952 --- docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7de4b8e43..aca89a8bc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,6 +9,7 @@ ADD src /invest/src ADD pyproject.toml /invest/pyproject.toml ADD setup.py /invest/setup.py ADD requirements.txt /invest/requirements.txt +ADD .git /invest/.git RUN cd /invest && python3 -m build # Create the container for distribution that has runtime dependencies.