Daily test using Terraform (#29)

1. Use terraform automate buy instance and run test
2. Support Aliyun and Opensource Redis-Like Database

.github/workflows/terraform.yaml

@@ -0,0 +1,79 @@
+name: Terraform in Daily Test
+    
+on:
+  schedule:
+    - cron: '0 18 * * *'  # 每天 UTC 时间 18:00，即中国时间凌晨 2 点
+    
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check GitHub Secrets
+        run: |
+          if [ -z "${{ secrets.MYAPP_GITHUB_TOKEN }}" ]; then
+            echo "GitHub Token is missing"
+          else
+            echo "GitHub Token is set"
+          fi
+
+          if [ -z "${{ secrets.MYAPP_USER_EMAIL }}" ]; then
+            echo "User Email is missing"
+          else
+            echo "User Email is set"
+          fi
+
+          if [ -z "${{ secrets.MYAPP_USER_NAME }}" ]; then
+            echo "User Name is missing"
+          else
+            echo "User Name is set"
+          fi
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          ref: dailyTest 
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v2
+        with:
+          terraform_version: 1.9.4 
+          
+      - name: Initialize Terraform
+        env:
+          TF_VAR_access_key: ${{ secrets.ALI_ACCESS_KEY }}
+          TF_VAR_secret_key: ${{ secrets.ALI_SECRET_KEY }}
+          TF_VAR_github_token: ${{ secrets.MYAPP_GITHUB_TOKEN }}
+          TF_VAR_user_name: ${{ secrets.MYAPP_USER_NAME }}
+          TF_VAR_user_email: ${{ secrets.MYAPP_USER_EMAIL }}
+        run: terraform -chdir=./Terraform/Aliyun init
+
+      - name: Apply Terraform Configuration
+        env:
+          TF_VAR_access_key: ${{ secrets.ALI_ACCESS_KEY }}
+          TF_VAR_secret_key: ${{ secrets.ALI_SECRET_KEY }}
+          TF_VAR_github_token: ${{ secrets.MYAPP_GITHUB_TOKEN }}
+          TF_VAR_user_name: ${{ secrets.MYAPP_USER_NAME }}
+          TF_VAR_user_email: ${{ secrets.MYAPP_USER_EMAIL }}
+        run: terraform -chdir=./Terraform/Aliyun apply -auto-approve
+        
+      - name: Print current directory and file list
+        run: |
+          cd ./Terraform/Aliyun
+          echo "Current directory:"
+          pwd
+          echo "Files in the current directory:"
+          ls
+
+      - name: Wait for 20 minutes before destroying resources
+        run: sleep 1200 # 等待 20 分钟
+        
+      - name: Destroy Terraform Configuration
+        env:
+          TF_VAR_access_key: ${{ secrets.ALI_ACCESS_KEY }}
+          TF_VAR_secret_key: ${{ secrets.ALI_SECRET_KEY }}
+          TF_VAR_github_token: ${{ secrets.MYAPP_GITHUB_TOKEN }}
+          TF_VAR_user_name: ${{ secrets.MYAPP_USER_NAME }}
+          TF_VAR_user_email: ${{ secrets.MYAPP_USER_EMAIL }}
+        run: terraform -chdir=./Terraform/Aliyun destroy -auto-approve
+

.gitignore

@@ -2,4 +2,4 @@
 test.json
 venv
 html
-__pycache__
+__pycache__

Terraform/Aliyun/main.tf

@@ -0,0 +1,338 @@
+variable "access_key" {
+  description = "Access key for Alicloud provider"
+  type        = string
+}
+
+variable "secret_key" {
+  description = "Secret key for Alicloud provider"
+  type        = string
+}
+
+variable "github_token" {
+  description = "GitHub token for accessing GitHub API"
+  type        = string
+}
+
+variable "user_name" {
+  description = "GitHub user name"
+  type        = string
+}
+
+variable "user_email" {
+  description = "GitHub user email"
+  type        = string
+}
+
+locals {
+  selected_zone_id = "cn-hongkong-b"
+}
+
+provider "alicloud" {
+  access_key = var.access_key
+  secret_key = var.secret_key
+  region     = "cn-hongkong"
+}
+
+# 创建 VPC
+resource "alicloud_vpc" "my_vpc" {
+  vpc_name   = "MyVPC"
+  cidr_block = "172.16.0.0/24"
+}
+
+# 创建 VSwitch
+resource "alicloud_vswitch" "my_vswitch" {
+  vswitch_name = "glcc_vswitch"
+  vpc_id       = alicloud_vpc.my_vpc.id
+  cidr_block   = "172.16.0.0/24"
+  zone_id      = local.selected_zone_id
+}
+
+resource "alicloud_security_group" "my_sg" {
+  name        = "glcc_test_security_group"
+  vpc_id      = alicloud_vpc.my_vpc.id
+  description = "Security Group for testing"
+}
+
+resource "alicloud_security_group_rule" "allow_ssh" {
+  security_group_id = alicloud_security_group.my_sg.id
+  type              = "ingress"
+  ip_protocol       = "tcp"
+  nic_type          = "intranet"
+  policy            = "accept"
+  port_range        = "22/22"
+  priority          = 1
+  cidr_ip           = "0.0.0.0/0"
+}
+
+resource "alicloud_security_group_rule" "allow_http" {
+  security_group_id = alicloud_security_group.my_sg.id
+  type              = "ingress"
+  ip_protocol       = "tcp"
+  nic_type          = "intranet"
+  policy            = "accept"
+  port_range        = "80/80"
+  priority          = 100
+  cidr_ip           = "0.0.0.0/0"
+}
+
+resource "alicloud_security_group_rule" "allow_https_outbound" {
+  type              = "egress"
+  security_group_id = alicloud_security_group.my_sg.id
+  ip_protocol       = "tcp"
+  nic_type          = "intranet"
+  policy            = "accept"
+  port_range        = "443/443"
+  cidr_ip           = "0.0.0.0/0"
+  description       = "Allow outbound HTTPS traffic to external services"
+}
+
+resource "alicloud_security_group_rule" "allow_redis" {
+  type              = "ingress"
+  security_group_id = alicloud_security_group.my_sg.id
+  ip_protocol       = "tcp"
+  nic_type          = "intranet"
+  policy            = "accept"
+  port_range        = "6379/17005"
+  cidr_ip           = "172.16.0.10/16"  # 仅允许特定 ECS 实例的私有 IP 访问
+  description       = "Allow inbound Redis traffic"
+}
+
+# 云原生 1GB 标准版 Tair 6.0
+resource "alicloud_kvstore_instance" "my_tair_standard" {
+  db_instance_name = "glcc_tair_standard"
+  instance_class   = "tair.rdb.1g"
+  instance_type    = "Redis"
+  engine_version   = "7.0"
+  zone_id          = local.selected_zone_id
+  vswitch_id       = alicloud_vswitch.my_vswitch.id
+  payment_type     = "PostPaid"
+  password         = "T123456@*"
+  security_ips     = ["172.16.0.10"]
+}
+
+
+# 云原生 1GB 集群版 Tair 6.0
+resource "alicloud_kvstore_instance" "my_tair_cluster" {
+  db_instance_name = "glcc_tair_cluster"
+  instance_class   = "tair.rdb.with.proxy.1g"
+  instance_type    = "Redis"
+  engine_version   = "7.0"
+  shard_count      = "8"
+  zone_id          = local.selected_zone_id
+  vswitch_id       = alicloud_vswitch.my_vswitch.id
+  payment_type     = "PostPaid"
+  password         = "T123456@*"
+  security_ips     = ["172.16.0.10"]
+}
+
+# 云原生 1GB 标准版 Redis 6.0
+resource "alicloud_kvstore_instance" "my_redis_standard" {
+  db_instance_name = "glcc_redis_standard"
+  instance_class   = "redis.shard.small.2.ce"
+  instance_type    = "Redis"
+  engine_version   = "7.0"
+  zone_id          = local.selected_zone_id
+  vswitch_id       = alicloud_vswitch.my_vswitch.id
+  payment_type     = "PostPaid"
+  password         = "T123456@*"
+  security_ips     = ["172.16.0.10"]
+}
+
+
+# 云原生 1GB 集群版 Redis 6.0
+resource "alicloud_kvstore_instance" "my_redis_cluster" {
+  db_instance_name = "glcc_redis_cluster"
+  instance_class   = "redis.shard.with.proxy.small.ce"
+  instance_type    = "Redis"
+  engine_version   = "7.0"
+  shard_count      = "8"
+  zone_id          = local.selected_zone_id
+  vswitch_id       = alicloud_vswitch.my_vswitch.id
+  payment_type     = "PostPaid"
+  password         = "T123456@*"
+  security_ips     = ["172.16.0.10"]
+}
+
+
+# 输出实例信息,目前用于验证
+output "tair_standard_instance_address" {
+  value = alicloud_kvstore_instance.my_tair_standard.connection_domain
+}
+
+output "tair_standard_instance_port" {
+  value = alicloud_kvstore_instance.my_tair_standard.private_connection_port
+}
+
+output "tair_standard_instance_password" {
+  value     = alicloud_kvstore_instance.my_tair_standard.password
+  sensitive = true
+}
+
+output "tair_cluster_instance_address" {
+  value = alicloud_kvstore_instance.my_tair_cluster.connection_domain
+}
+
+output "tair_cluster_instance_port" {
+  value = alicloud_kvstore_instance.my_tair_cluster.private_connection_port
+}
+
+output "tair_cluster_instance_password" {
+  value     = alicloud_kvstore_instance.my_tair_cluster.password
+  sensitive = true
+}
+
+output "redis_standard_instance_address" {
+  value = alicloud_kvstore_instance.my_redis_standard.connection_domain
+}
+
+output "redis_standard_instance_port" {
+  value = alicloud_kvstore_instance.my_redis_standard.private_connection_port
+}
+
+output "redis_standard_instance_password" {
+  value     = alicloud_kvstore_instance.my_redis_standard.password
+  sensitive = true
+}
+
+output "redis_cluster_instance_address" {
+  value = alicloud_kvstore_instance.my_redis_cluster.connection_domain
+}
+
+output "redis_cluster_instance_port" {
+  value = alicloud_kvstore_instance.my_redis_cluster.private_connection_port
+}
+
+output "redis_cluster_instance_password" {
+  value     = alicloud_kvstore_instance.my_redis_cluster.password
+  sensitive = true
+}
+
+# 创建 ECS 实例
+resource "alicloud_instance" "my_ecs" {
+  private_ip               = "172.16.0.10"
+  instance_type            = "ecs.g6.xlarge"
+  security_groups          = [alicloud_security_group.my_sg.id]
+  instance_charge_type     = "PostPaid"
+  internet_charge_type     = "PayByTraffic"
+  internet_max_bandwidth_out = 10
+  image_id                 = "ubuntu_22_04_x64_20G_alibase_20240130.vhd"
+  instance_name            = "glcc_ecs"
+  vswitch_id               = alicloud_vswitch.my_vswitch.id
+  system_disk_category     = "cloud_efficiency"
+  password                 = "T123456@*"
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  user_data = <<EOF
+#!/bin/bash
+export HOME=/root
+apt-get update
+apt-get install -y python3-pip git docker.io
+systemctl start docker
+pip install "pyyaml>=6.0"
+git config --global credential.helper 'store'
+source /etc/profile
+
+# 写入数据库配置信息
+cat <<EOT >> /root/db_config.yml
+AliyunTair:
+  host: ${alicloud_kvstore_instance.my_tair_standard.connection_domain}
+  port: ${alicloud_kvstore_instance.my_tair_standard.private_connection_port}
+  password: T123456@*
+  ssl: false
+  cluster: false
+  version: 7.0
+
+AliyunTairCluster:
+  host: ${alicloud_kvstore_instance.my_tair_cluster.connection_domain}
+  port: ${alicloud_kvstore_instance.my_tair_cluster.private_connection_port}
+  password: T123456@*
+  ssl: false
+  cluster: true
+  version: 7.0
+
+AliyunRedis:
+  host: ${alicloud_kvstore_instance.my_redis_standard.connection_domain}
+  port: ${alicloud_kvstore_instance.my_redis_standard.private_connection_port}
+  password: T123456@*
+  ssl: false
+  cluster: false
+  version: 7.0
+
+AliyunRedisCluster:
+  host: ${alicloud_kvstore_instance.my_redis_cluster.connection_domain}
+  port: ${alicloud_kvstore_instance.my_redis_cluster.private_connection_port}
+  password: T123456@*
+  ssl: false
+  cluster: true
+  version: 7.0
+EOT
+
+
+# 拉取和运行数据库容器
+docker pull docker.io/redis:latest
+docker run --name redis -d -p 6379:6379 redis
+
+docker pull docker.dragonflydb.io/dragonflydb/dragonfly:latest
+docker run --name dragonflydb -d -p 6380:6379 docker.dragonflydb.io/dragonflydb/dragonfly
+
+docker pull apache/kvrocks:latest
+docker run --name kvrocks -d -p 6381:6666 apache/kvrocks
+
+docker pull docker.io/eqalpha/keydb:latest
+docker run --name keydb -d -p 6382:6379 eqalpha/keydb
+
+docker pull docker.io/pikadb/pika:latest
+docker run --name pika -d -p 6383:6383 pikadb/pika
+
+docker pull valkey/valkey:latest
+docker run --name valkey -d -p 6384:6379 valkey/valkey
+
+
+# 配置Git
+echo "https://${var.github_token}:x-oauth-basic@github.com" > ~/.git-credentials
+git config --global user.name "${var.user_name}"
+git config --global user.email "${var.user_email}"
+
+git clone https://github.com/redis/redis.git
+cd redis
+make -j
+cd utils/create-cluster
+ ./create-cluster start
+yes yes | ./create-cluster create
+
+# 可替换为含有dailyTest分支仓库的url
+REPO_URL="https://github.com/tair-opensource/resp-compatibility.git"
+RETRY_COUNT=0
+MAX_RETRIES=10
+SLEEP_DURATION=30
+
+while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
+  if git clone $REPO_URL; then
+    echo "Git clone succeeded"
+    break
+  else
+    RETRY_COUNT=$((RETRY_COUNT + 1))
+    echo "Git clone failed, attempt $RETRY_COUNT/$MAX_RETRIES"
+    sleep $SLEEP_DURATION
+  fi
+done
+
+if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
+  echo "Git clone failed after $MAX_RETRIES attempts" >&2
+  exit 1
+fi
+
+cd resp-compatibility
+git checkout dailyTest
+python3 conn.py
+EOF
+}
+
+
+output "ecs_ip_address" {
+  value = alicloud_instance.my_ecs.private_ip
+}
+

config.yaml

@@ -14,31 +14,39 @@ SpecificVersion:
 # the host, port, password of all the db to be tested
 Database:
   Redis:
-    host: 127.0.0.1
+    host: 172.16.0.10
     port: 6379
     password:
     ssl: false
     cluster: false
-    version: unstable
+    version: latest
-
+  
-  DragonflyDB:
+  RedisCluster:
     host: 127.0.0.1
-    port: 6380
+    port: 30001 
+    password:
+    ssl: false
+    cluster: true
+    version: latest
+  
+  Valkey:
+    host: 172.16.0.10
+    port: 6384
     password:
     ssl: false
     cluster: false
     version: latest
 
   Kvrocks:
-    host: 127.0.0.1
+    host: 172.16.0.10
     port: 6381
     password:
     ssl: false
     cluster: false
-    version: 2.2.0
+    version: latest
 
   KeyDB:
-    host: 127.0.0.1
+    host: 172.16.0.10
     port: 6382
     password:
     ssl: false
@@ -46,29 +54,13 @@ Database:
     version: latest
 
   Pika:
-    host: 127.0.0.1
+    host: 172.17.0.6
-    port: 6383
+    port: 9221
     password:
     ssl: false
     cluster: false
     version: latest
 
-  Tair:
-    host:
-    port:
-    password:
-    ssl:
-    cluster: false
-    version:
-
-  Kvstore:
-    host:
-    port:
-    password:
-    ssl:
-    cluster: false
-    version:
-
   ElastiCache:
     host:
     port:
@@ -115,4 +107,4 @@ Database:
     password:
     ssl:
     cluster: false
-    version:
+    version:

conn.py

@@ -0,0 +1,102 @@
+import os
+import threading
+import time
+import yaml
+import subprocess
+
+# 更新配置文件
+try:
+    yml_file_path = '/root/db_config.yml'
+    config_file_path = 'config.yaml'
+
+    with open(yml_file_path, 'r') as yml_file, open(config_file_path, 'r') as config_file:
+        yml_data = yaml.safe_load(yml_file)
+        config_data = yaml.safe_load(config_file)
+
+        for db_name, db_config in yml_data.items():
+            if db_name in config_data['Database']:
+                for key, value in db_config.items():
+                    config_data['Database'][db_name][key] = value
+            else:
+                config_data['Database'][db_name] = db_config
+
+    with open(config_file_path, 'w') as config_file:
+        yaml.dump(config_data, config_file, default_flow_style=False)
+
+    print("Config file updated successfully.")
+
+except FileNotFoundError as e:
+    print(f"Error: {e}. Please check the file paths.")
+except yaml.YAMLError as e:
+    print(f"Error in YAML processing: {e}")
+except Exception as e:
+    print(f"An unexpected error occurred: {e}")
+
+
+# 执行命令并判断是否成功
+def execute_command(commands):
+    for command in commands:
+        result = subprocess.run(command, shell=True, capture_output=True, text=True)
+        if result.returncode != 0:
+            print(f"Error executing command '{command}': {result.stderr}")
+            return False
+        else:
+            print(f"Successfully executed command '{command}': {result.stdout}")
+    return True
+
+
+
+commands = [
+    "apt-get update",
+    "apt-get install -y python3-pip",
+]
+if not execute_command(commands):
+    print("Failed to update or install packages. Exiting...")
+    exit(1)
+
+# 运行测试命令
+run_test_command = [
+    "pip3 install -r requirements.txt",
+    "python3 resp_compatibility.py --testfile cts.json --genhtml --show-failed",
+]
+
+def run_test():
+    if not execute_command(run_test_command):
+        print("Test failed. Exiting...")
+        exit(1)
+    else:
+        print("Test completed successfully.")
+
+# 启动测试脚本的线程
+test_thread = threading.Thread(target=run_test)
+test_thread.start()
+
+time.sleep(300)
+
+# 提交和推送测试结果
+commit_and_push_commands = [
+    "mv html /root",
+    "git stash -u",
+    "git checkout gh-pages || git checkout -b gh-pages",
+    "git pull origin gh-pages",
+    "cp -r /root/html/* .", 
+    "git add .",
+    "git commit -m 'Update test results'",
+]
+if not execute_command(commit_and_push_commands):
+    print("Failed to commit and push changes. Exiting...")
+    exit(1)
+
+# 推送到 GitHub 并重试
+def git_push_with_retry():
+    while True:
+        result = subprocess.run("git push -u origin gh-pages", shell=True, capture_output=True, text=True)
+        if result.returncode == 0:
+            print("Successfully pushed to GitHub.")
+            break
+        else:
+            print(f"Git push failed: {result.stderr}. Retrying in 5 seconds...")
+            time.sleep(5)
+
+git_push_with_retry()
+

resp_compatibility.py

@@ -220,40 +220,60 @@ def generate_html_report(logdir, configs):
     html.write("This page is automatically generated by <a href=\"https://github.com/tair-opensource/"
                "compatibility-test-suite-for-redis\">compatibility-test-suite-for-redis</a> "
                "to show the compatibility of the following Redis-Like systems and different versions of Redis.<br><br>")
-    html.write("<table>")
+
-    # generate header
+    # Separate databases into cluster and standalone
-    html.write("<thead>")
+    cluster_databases = []
-    html.write("<tr>")
+    standalone_databases = []
-    html.write("<th>Product / Redis Version</th>")
+
-    for version in configs['SpecificVersion']:
-        html.write(f"<th>{version}</th>")
-    html.write("</tr>")
-    html.write("</thead>")
-    # generate body
-    html.write("<tbody>")
     for config in configs['Database']:
+        if configs['Database'][config]['cluster']:
+            cluster_databases.append(config)
+        else:
+            standalone_databases.append(config)
+
+    # Function to generate a table
+    def generate_table(databases, title):
+        html.write(f"<h3>{title}</h3>")
+        html.write("<table>")
+        # generate header
+        html.write("<thead>")
         html.write("<tr>")
-        html.write(f"<td>{config}({configs['Database'][config]['version']})</td>")
+        html.write("<th>Product / Redis Version</th>")
         for version in configs['SpecificVersion']:
-            filepath = f"{logdir}/{config}-{version}.html"
+            html.write(f"<th>{version}</th>")
-            if not os.path.exists(filepath):
-                html.write(f"<td>-</td>")
-                continue
-            with open(filepath, 'r') as f:
-                s = f.read()
-            match = re.search(r"rate: (\d+\.\d+)%", s)
-            assert match
-            rate = match.group(1)
-            color = "#40de5a"
-            if eval(rate) < 80:
-                color = "#f05654"
-            elif eval(rate) < 100:
-                color = "#ffa400"
-            html.write(f"<td style=\"background:{color}\">{rate}% <a href=\"{config}-{version}.html\">detail</a></td>")
         html.write("</tr>")
-    html.write("</tbody>")
+        html.write("</thead>")
-    html.write("</table>")
+        # generate body
-    html.write("<br>")
+        html.write("<tbody>")
+        for config in databases:
+            html.write("<tr>")
+            html.write(f"<td>{config}({configs['Database'][config]['version']})</td>")
+            for version in configs['SpecificVersion']:
+                filepath = f"{logdir}/{config}-{version}.html"
+                if not os.path.exists(filepath):
+                    html.write(f"<td>-</td>")
+                    continue
+                with open(filepath, 'r') as f:
+                    s = f.read()
+                match = re.search(r"rate: (\d+\.\d+)%", s)
+                assert match
+                rate = match.group(1)
+                color = "#40de5a"
+                if eval(rate) < 80:
+                    color = "#f05654"
+                elif eval(rate) < 100:
+                    color = "#ffa400"
+                html.write(f"<td style=\"background:{color}\">{rate}% <a href=\"{config}-{version}.html\">detail</a></td>")
+            html.write("</tr>")
+        html.write("</tbody>")
+        html.write("</table><br>")
+
+    # Generate standalone table
+    generate_table(standalone_databases, "Standalone Databases")
+
+    # Generate cluster table
+    generate_table(cluster_databases, "Cluster Databases")
+
     time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
     html.write(f"This report was generated on {time}.")
     html.write("<style>table {border-collapse: collapse;} th, td {border: 1px solid black; padding: 8px;}</style>")