Add self-hosted runner support (#199)

* ci: add testsz for self hosted runner

* ci: use self-hosted runner

* ci: run ci on push

* ci: remove cd to tmp dir

* ci: fix

* ci: add environment back

* ci: set concurrency

* ci: fix build env image

* Update .github/workflows/test-arealite.yml

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update ci/build_env_image.sh

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* ci: add some test code

* ci: simplify

* ci: try to fix

* ci: test

* ci: test2

* ci: test3

* ci: test3

* ci: tidy up

* ci: simplify

* ci: do not trigger on push

* ci: test arealite/tests/

* ci: use UUID as run id

* ci: use openssl rand to genreate run id

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

.github/workflows/test-arealite.yml

@@ -13,42 +13,8 @@ jobs:
   test-arealite:
     environment:
       name: AReaLite-unittests
-    runs-on: ubuntu-latest
-    concurrency:
-      group: test-arealite
+    runs-on: gpu
     steps:
       - uses: actions/checkout@v4
 
-      - uses: appleboy/ssh-action@v1
-        env:
-          GIT_REPO_URL: https://github.bibk.top/${{ github.repository }}
-          GIT_COMMIT_SHA: ${{ github.sha }}
-          CI_NODE_SUDO_PW: ${{ secrets.CI_NODE_SUDO_PW }}
-        with:
-          host: ${{ secrets.CI_NODE_ADDR }}
-          username: ${{ secrets.CI_NODE_USER }}
-          key: ${{ secrets.REMOTE_SSH_KEY }}
-          envs: GIT_REPO_URL,GIT_COMMIT_SHA,CI_NODE_SUDO_PW
-          script_path: ci/clone_repo.sh
-
-      - uses: appleboy/ssh-action@v1
-        env:
-          GIT_COMMIT_SHA: ${{ github.sha }}
-        with:
-          host: ${{ secrets.CI_NODE_ADDR }}
-          username: ${{ secrets.CI_NODE_USER }}
-          key: ${{ secrets.REMOTE_SSH_KEY }}
-          command_timeout: 2h
-          envs: GIT_COMMIT_SHA
-          script_path: ci/build_env_image.sh
-
-      - uses: appleboy/ssh-action@v1
-        env:
-          GIT_COMMIT_SHA: ${{ github.sha }}
-        with:
-          host: ${{ secrets.CI_NODE_ADDR }}
-          username: ${{ secrets.CI_NODE_USER }}
-          key: ${{ secrets.REMOTE_SSH_KEY }}
-          command_timeout: 1h
-          envs: GIT_COMMIT_SHA
-          script_path: ci/test_arealite.sh
+      - run: bash ./ci/test_arealite.sh

ci/build_env_image.sh

@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-GIT_COMMIT_SHA=${GIT_COMMIT_SHA:?"GIT_COMMIT_SHA is not set"}
-
-echo "GIT_COMMIT_SHA: $GIT_COMMIT_SHA"
-
-RUN_ID="areal-$GIT_COMMIT_SHA"
-cd "/tmp/$RUN_ID"
-
-# If there is already an image for the current environment, skip the build.
-ENV_SHA=$(sha256sum pyproject.toml | awk '{print $1}')
-if docker images --format '{{.Repository}}:{{.Tag}}' | grep -q "areal-env:$ENV_SHA"; then
-    echo "Image areal-env already exists, skipping build."
-    exit 0
-fi
-
-if docker ps -a --format '{{.Names}}' | grep -q "$RUN_ID"; then
-    docker rm -f $RUN_ID
-fi
-
-docker run \
-    --name $RUN_ID \
-    --gpus all \
-    --shm-size=8g \
-    -v $(pwd):/workspace \
-    -w /workspace \
-    nvcr.io/nvidia/pytorch:25.01-py3 \
-    bash -c "
-        python -m pip install --upgrade pip
-        pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
-        pip config unset global.extra-index-url
-        bash examples/env/scripts/setup-pip-deps.sh
-        pip uninstall -y transformer-engine
-        mv ./sglang /sglang
-    " || { docker rm -f $RUN_ID; exit 1; }
-
-docker commit $RUN_ID "areal-env:$ENV_SHA"
-docker rm -f $RUN_ID

ci/clone_repo.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-GIT_REPO_URL=${GIT_REPO_URL:?"GIT_REPO_URL is not set"}
-GIT_COMMIT_SHA=${GIT_COMMIT_SHA:?"GIT_COMMIT_SHA is not set"}
-
-echo "GIT_REPO_URL: $GIT_REPO_URL"
-echo "GIT_COMMIT_SHA: $GIT_COMMIT_SHA"
-
-RUN_ID="areal-$GIT_COMMIT_SHA"
-echo ${CI_NODE_SUDO_PW} | sudo -S rm -rf "/tmp/$RUN_ID"
-mkdir -p "/tmp/$RUN_ID"
-cd "/tmp/$RUN_ID"
-
-git init
-git remote add origin "$GIT_REPO_URL"
-git fetch --depth 1 origin "$GIT_COMMIT_SHA"
-git checkout FETCH_HEAD

ci/test_arealite.sh

@@ -2,28 +2,46 @@
 
 set -e
 
-GIT_COMMIT_SHA=${GIT_COMMIT_SHA:?"GIT_COMMIT_SHA is not set"}
+RUN_ID="test-arealite-$(openssl rand -hex 6)"
 
-echo "GIT_COMMIT_SHA: $GIT_COMMIT_SHA"
+# Calculate environment hash from pyproject.toml
+ENV_SHA=$(sha256sum pyproject.toml | awk '{print $1}')
 
-RUN_ID="areal-$GIT_COMMIT_SHA"
-cd "/tmp/$RUN_ID"
+# Build environment image if it doesn't exist
+if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -q "areal-env:$ENV_SHA"; then
+    echo "Building image areal-env:$ENV_SHA..."
 
-if docker ps -a --format '{{.Names}}' | grep -q "$RUN_ID"; then
+    # Build the environment image
+    docker run \
+        --name $RUN_ID \
+        -v $(pwd):/workspace \
+        -w /workspace \
+        nvcr.io/nvidia/pytorch:25.01-py3 \
+        bash -c "
+            pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+            pip config unset global.extra-index-url
+            pip uninstall -y --ignore-installed transformer-engine
+            pip uninstall -y --ignore-installed torch-tensorrt
+            pip uninstall -y --ignore-installed nvidia-dali-cuda120
+            bash examples/env/scripts/setup-pip-deps.sh
+        " || { docker rm -f $RUN_ID; exit 1; }
+
+    # Commit the container as the environment image
+    docker commit $RUN_ID areal-env:$ENV_SHA
     docker rm -f $RUN_ID
+else
+    echo "Image areal-env:$ENV_SHA already exists, skipping build."
 fi
 
-ENV_SHA=$(sha256sum pyproject.toml | awk '{print $1}')
+# Run tests using the environment image
+echo "Running tests on image areal-env:$ENV_SHA..."
 docker run \
+    -e HF_ENDPOINT=https://hf-mirror.com \
     --name $RUN_ID \
     --gpus all \
     --shm-size=8g \
+    --rm \
     -v $(pwd):/workspace \
     -w /workspace \
-    "areal-env:$ENV_SHA" \
-    bash -c "
-        mv /sglang ./sglang
-        HF_ENDPOINT=https://hf-mirror.com python -m pytest -s arealite/
-    " || { docker rm -f $RUN_ID; exit 1; }
-
-docker rm -f $RUN_ID
+    areal-env:$ENV_SHA \
+    python -m pytest -s arealite/tests/