feat: Add docker support (#99)

* feat:Add Docker Support * translate
2025-03-30 15:15:32 +08:00 · 2025-03-30 15:15:32 +08:00 · f64ddc108b
parent 391d488ee7
commit f64ddc108b
9 changed files with 315 additions and 37 deletions
--- a/Dockerfile.backend
+++ b/Dockerfile.backend
@ -0,0 +1,65 @@
+FROM python:3.12
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies, Poetry and configure it
+RUN apt-get update && apt-get install -y \
+    build-essential cmake git curl wget lsof vim unzip sqlite3 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && pip install --upgrade pip \
+    && pip install poetry \
+    && poetry config virtualenvs.create false
+
+# Create directories
+RUN mkdir -p /app/dependencies /app/data/sqlite /app/data/chroma_db /app/logs /app/run /app/resources
+
+# Copy dependency files - Files that rarely change
+COPY dependencies/graphrag-1.2.1.dev27.tar.gz /app/dependencies/
+COPY dependencies/llama.cpp.zip /app/dependencies/
+
+# Build llama.cpp
+RUN LLAMA_LOCAL_ZIP="dependencies/llama.cpp.zip" \
+    && echo "Using local llama.cpp archive..." \
+    && unzip -q "$LLAMA_LOCAL_ZIP" \
+    && cd llama.cpp \
+    && mkdir -p build && cd build \
+    && cmake .. \
+    && cmake --build . --config Release \
+    && if [ ! -f "bin/llama-server" ]; then \
+         echo "Build failed: llama-server executable not found" && exit 1; \
+       else \
+         echo "Successfully built llama-server"; \
+       fi
+
+# 
+# Copy project configuration - Files that occasionally change
+COPY pyproject.toml README.md /app/
+
+RUN poetry install --no-interaction --no-root \
+    && pip install --force-reinstall dependencies/graphrag-1.2.1.dev27.tar.gz
+
+
+# Copy source code - Files that frequently change
+COPY docker/ /app/docker/
+COPY lpm_kernel/ /app/lpm_kernel/
+
+# Check module import
+RUN python -c "import lpm_kernel; print('Module import check passed')"
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONPATH=/app \
+    BASE_DIR=/app/data \
+    LOCAL_LOG_DIR=/app/logs \
+    RUN_DIR=/app/run \
+    RESOURCES_DIR=/app/resources \
+    APP_ROOT=/app \
+    FLASK_APP=lpm_kernel.app
+
+# Expose ports
+EXPOSE 8002 8080
+
+# Set the startup command
+CMD ["bash", "-c", "echo \"Checking SQLite database...\" && if [ ! -s /app/data/sqlite/lpm.db ]; then echo \"SQLite database not found or empty, initializing...\" && mkdir -p /app/data/sqlite && sqlite3 /app/data/sqlite/lpm.db \".read /app/docker/sqlite/init.sql\" && echo \"SQLite database initialized successfully\" && echo \"Tables created:\" && sqlite3 /app/data/sqlite/lpm.db \".tables\"; else echo \"SQLite database already exists, skipping initialization\"; fi && echo \"Checking ChromaDB...\" && if [ ! -d /app/data/chroma_db/documents ] || [ ! -d /app/data/chroma_db/document_chunks ]; then echo \"ChromaDB collections not found, initializing...\" && python /app/docker/app/init_chroma.py && echo \"ChromaDB initialized successfully\"; else echo \"ChromaDB already exists, skipping initialization\"; fi && echo \"Starting application at $(date)\" >> /app/logs/backend.log && cd /app && python -m flask run --host=0.0.0.0 --port=${LOCAL_APP_PORT:-8002} >> /app/logs/backend.log 2>&1"]
--- a/Dockerfile.frontend
+++ b/Dockerfile.frontend
@ -0,0 +1,25 @@
+FROM node:23
+
+# Set working directory
+WORKDIR /app
+
+# Copy frontend package files
+COPY lpm_frontend/package.json lpm_frontend/package-lock.json* /app/
+
+# Install dependencies
+RUN npm install
+
+# Copy frontend code
+COPY lpm_frontend/ /app/
+
+# Set environment variable for backend URL (can be overridden in docker-compose)
+ENV DOCKER_API_BASE_URL=http://backend:8002
+
+# Create logs directory
+RUN mkdir -p /app/logs
+
+# Expose frontend port
+EXPOSE 3000
+
+# Start frontend service
+CMD ["npm", "run", "dev"]
--- a/48
+++ b/48
@ -9,7 +9,7 @@
 #   make restart-force - Force restart and reset data
 #   make status       - Show status of all services

-.PHONY: install test format lint all setup start stop restart restart-backend restart-force help check-conda check-env
+.PHONY: install test format lint all setup start stop restart restart-backend restart-force help check-conda check-env docker-build docker-up docker-down docker-build-backend docker-build-frontend docker-restart-backend docker-restart-frontend docker-restart-all

 # Show help message
 help:
@ -33,6 +33,16 @@ help:
 	@echo "  make restart-force         - Force restart and reset data"
 	@echo "  make status                - Show status of all services"
 	@echo ""
+	@echo "\033[1;32m▶ DOCKER COMMANDS:\033[0m"
+	@echo "  make docker-build          - Build all Docker images"
+	@echo "  make docker-up             - Start all Docker containers"
+	@echo "  make docker-down           - Stop all Docker containers"
+	@echo "  make docker-build-backend  - Build only backend Docker image"
+	@echo "  make docker-build-frontend - Build only frontend Docker image"
+	@echo "  make docker-restart-backend - Restart only backend container"
+	@echo "  make docker-restart-frontend - Restart only frontend container"
+	@echo "  make docker-restart-all    - Restart all Docker containers"
+	@echo ""
 	@echo "\033[1mAll Available Commands:\033[0m"
 	@echo "  make help                  - Show this help message"
 	@echo "  make check-env             - Check environment without installing"
@ -77,6 +87,42 @@ restart-force:
 status:
 	zsh ./scripts/status.sh

+# Docker commands
+# Set Docker environment variable for all Docker commands
+docker-%: export IN_DOCKER_ENV=1
+docker-build:
+	docker-compose build
+
+docker-up:
+	docker-compose up -d
+
+docker-down:
+	docker-compose down
+
+docker-build-backend:
+	docker-compose build backend
+
+docker-build-frontend:
+	docker-compose build frontend
+
+docker-restart-backend:
+	docker-compose stop backend
+	docker-compose rm -f backend
+	docker-compose build backend || { echo "\033[1;31m❌ Backend build failed! Aborting operation...\033[0m"; exit 1; }
+	docker-compose up -d backend
+
+docker-restart-frontend:
+	docker-compose stop frontend
+	docker-compose rm -f frontend
+	docker-compose build frontend || { echo "\033[1;31m❌ Frontend build failed! Aborting operation...\033[0m"; exit 1; }
+	docker-compose up -d frontend
+
+docker-restart-all:
+	docker-compose stop
+	docker-compose rm -f
+	docker-compose build || { echo "\033[1;31m❌ Build failed! Aborting operation...\033[0m"; exit 1; }
+	docker-compose up -d
+
 # Commands that require conda environment
 install: check-conda
 	poetry install
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -0,0 +1,64 @@
+version: '3.8'
+
+services:
+  backend:
+    build:
+      context: .
+      dockerfile: Dockerfile.backend
+    container_name: second-me-backend
+    restart: unless-stopped
+    ports:
+      - "8002:8002"
+      - "8080:8080"
+    volumes:
+      - ./data:/app/data
+      - ./logs:/app/logs
+      - ./run:/app/run
+      - ./resources:/app/resources
+      - ./docker:/app/docker
+      - ./.env:/app/.env
+    environment:
+      # Environment variables
+      - LOCAL_APP_PORT=8002
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    deploy:
+      resources:
+        limits:
+          # Set container memory limit to 24GB
+          memory: 24G
+        reservations:
+          # Memory reservation
+          memory: 6G
+    networks:
+      - second-me-network
+
+  frontend:
+    build:
+      context: .
+      dockerfile: Dockerfile.frontend
+    container_name: second-me-frontend
+    restart: unless-stopped
+    ports:
+      - "3000:3000"
+    volumes:
+      - ./logs:/app/logs
+      - ./resources:/app/resources
+    environment:
+      - VITE_API_BASE_URL=http://backend:8002
+    depends_on:
+      - backend
+    deploy:
+      resources:
+        limits:
+          # Set container memory limit to 8GB
+          memory: 8G
+        reservations:
+          # Memory reservation
+          memory: 2G
+    networks:
+      - second-me-network
+
+networks:
+  second-me-network:
+    driver: bridge
--- a/lpm_frontend/next.config.js
+++ b/lpm_frontend/next.config.js
@ -1,7 +1,8 @@
 const nextConfig = {
  reactStrictMode: false,
  async rewrites() {
-    const API_URL = `${process.env.HOST_ADDRESS || 'http://127.0.0.1'}:${process.env.LOCAL_APP_PORT || 8002}`;
+    const dockerApiBaseUrl = process.env.DOCKER_API_BASE_URL;
+    const localApiBaseUrl = `${process.env.HOST_ADDRESS || 'http://127.0.0.1'}:${process.env.LOCAL_APP_PORT || 8002}`;

    return [
      {
@ -10,7 +11,9 @@ const nextConfig = {
      },
      {
        source: '/api/:path*',
-        destination: `${API_URL}/api/:path*`
+        destination: dockerApiBaseUrl
+          ? `${dockerApiBaseUrl}/api/:path*`
+          : `${localApiBaseUrl}/api/:path*`
      }
    ];
  },
--- a/lpm_kernel/L2/merge_weights_for_user.sh
+++ b/lpm_kernel/L2/merge_weights_for_user.sh
@ -1,3 +1,5 @@
+#!/bin/sh
+
 python lpm_kernel/L2/merge_lora_weights.py \
 --base_model_path "${MODEL_BASE_PATH}" \
 --lora_adapter_path "${MODEL_PERSONAL_DIR}" \
--- a/lpm_kernel/api/common/script_executor.py
+++ b/lpm_kernel/api/common/script_executor.py
@ -13,9 +13,16 @@ logger.error("ERROR: ScriptExecutor module loaded")

 class ScriptExecutor:
    def __init__(self):
-        self.conda_env = os.getenv("CONDA_DEFAULT_ENV")
-        if not self.conda_env:
-            raise ValueError("CONDA_DEFAULT_ENV environment variable is not set")
+        # Check if running in Docker environment
+        self.in_docker = os.getenv("IN_DOCKER_ENV") == "1" or os.path.exists("/.dockerenv")
+        
+        # Only check conda environment if not in Docker
+        if not self.in_docker:
+            self.conda_env = os.getenv("CONDA_DEFAULT_ENV")
+            if not self.conda_env:
+                raise ValueError("CONDA_DEFAULT_ENV environment variable is not set and not running in Docker")
+        else:
+            self.conda_env = "docker-env"  # Use a placeholder for Docker

    def execute(
        self,
@ -26,7 +33,7 @@ class ScriptExecutor:
        log_file: Optional[str] = None,
    ) -> Dict[str, Any]:
        """
-        Execute scripts in the specified conda environment
+        Execute scripts in the specified conda environment or directly in Docker

        Args:
            script_path: Script path or command
@ -40,31 +47,39 @@ class ScriptExecutor:
        """
        try:
            # Build the complete command
-            if script_path.endswith(".py"):
-                # Python script
-                cmd = [
-                    "conda",
-                    "run",
-                    "-n",
-                    self.conda_env,
-                    "python",
-                    "-u",
-                    script_path,
-                ]  # Add -u parameter to disable output buffering
-            elif script_path.endswith(".sh"):
-                # Shell script
-                cmd = [
-                    "conda",
-                    "run",
-                    "-n",
-                    self.conda_env,
-                    "bash",
-                    "-x",
-                    script_path,
-                ]  # Add -x parameter to display executed commands
+            if self.in_docker:
+                # In Docker, directly execute Python or the command
+                if script_path.endswith(".py"):
+                    cmd = ["python", script_path]
+                else:
+                    cmd = [script_path]
            else:
-                # Other commands
-                cmd = ["conda", "run", "-n", self.conda_env, script_path]
+                # In conda environment
+                if script_path.endswith(".py"):
+                    # Python script
+                    cmd = [
+                        "conda",
+                        "run",
+                        "-n",
+                        self.conda_env,
+                        "python",
+                        "-u",
+                        script_path,
+                    ]  # Add -u parameter to disable output buffering
+                elif script_path.endswith(".sh"):
+                    # Shell script
+                    cmd = [
+                        "conda",
+                        "run",
+                        "-n",
+                        self.conda_env,
+                        "bash",
+                        "-x",
+                        script_path,
+                    ]  # Add -x parameter to display executed commands
+                else:
+                    # Other commands
+                    cmd = ["conda", "run", "-n", self.conda_env, script_path]

            # Add additional parameters
            if args:
--- a/lpm_kernel/api/common/script_runner.py
+++ b/lpm_kernel/api/common/script_runner.py
@ -43,6 +43,55 @@ class ScriptRunner:
        """
        return os.environ.get("CONDA_DEFAULT_ENV")

+    def _check_execution_env(self) -> Dict[str, str]:
+        """
+        Get current execution environment information, supporting conda, docker or regular system environment
+        Returns:
+            Dict[str, str]: Dictionary containing environment type and detailed information
+        """
+        env_info = {
+            "type": "system",
+            "details": "Unknown environment"
+        }
+        
+        # Check if in conda environment
+        conda_env = self._check_conda_env()
+        if conda_env:
+            env_info["type"] = "conda"
+            env_info["details"] = conda_env
+            return env_info
+        
+        # Check if in docker environment - first check environment variable
+        if os.environ.get("IN_DOCKER_ENV") == "1":
+            env_info["type"] = "docker"
+            env_info["details"] = "docker-env-variable"
+            return env_info
+        
+        # Then check if .dockerenv file exists
+        if os.path.exists("/.dockerenv"):
+            env_info["type"] = "docker"
+            container_id = "unknown"
+            try:
+                with open("/proc/self/cgroup", "r") as f:
+                    for line in f:
+                        if "docker" in line:
+                            container_id = line.split("/")[-1].strip()
+                            break
+            except Exception:
+                pass
+            env_info["details"] = f"container:{container_id}"
+            return env_info
+        
+        # Regular system environment
+        try:
+            import platform
+            system_info = platform.platform()
+            env_info["details"] = system_info
+        except Exception:
+            pass
+            
+        return env_info
+
    def _check_python_version(self) -> str:
        """
        Get Python version information
@ -73,11 +122,10 @@ class ScriptRunner:
            if not os.path.exists(script_path):
                raise FileNotFoundError(f"Script does not exist: {script_path}")

-            # Get conda environment
-            conda_env = self._check_conda_env()
-            if not conda_env:
-                raise EnvironmentError("Unable to get conda environment information")
-
+            # Get execution environment information
+            env_info = self._check_execution_env()
+            logger.info(f"Running in environment: {env_info['type']} ({env_info['details']})")
+            
            # Prepare log file
            log_file = self.base_log_path
            logger.info(f"Starting {script_type} task, log file: {log_file}")
@ -139,7 +187,7 @@ class ScriptRunner:

            return {
                "pid": pid,
-                "conda_env": conda_env,
+                "environment": env_info,
                "log_file": log_file,
                "exit_code": exit_code,
            }
--- a/scripts/start.sh
+++ b/scripts/start.sh
@ -230,6 +230,16 @@ start_services() {
        
        log_info "Starting frontend service..."
        cd lpm_frontend
+
+        # Copy environment variables from root directory to frontend directory
+        log_info "Copying environment variables to frontend directory..."
+        if [[ -f "../.env" ]]; then
+            # Extract required environment variables and create frontend .env file
+            grep -E "^(HOST_ADDRESS|LOCAL_APP_PORT)=" "../.env" > .env
+            log_success "Environment variables copied to frontend .env file"
+        else
+            log_warning "Root directory .env file does not exist, cannot copy environment variables"
+        fi
        
        # Copy environment variables from root directory to frontend directory
        log_info "Copying environment variables to frontend directory..."