From f7a1c63324a56b5f96a829f373ab0a1b4a524617 Mon Sep 17 00:00:00 2001
From: "bodong.ybd" <bodong.ybd@alibaba-inc.com>
Date: Thu, 27 Apr 2023 13:55:48 +0800
Subject: [PATCH] Init commit

---
 .gitignore                  |   3 +
 README.md                   |  41 +++
 cts.json                    | 496 ++++++++++++++++++++++++++++++++++++
 redis_compatibility_test.py | 207 +++++++++++++++
 requirements.txt            |   1 +
 5 files changed, 748 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 README.md
 create mode 100644 cts.json
 create mode 100644 redis_compatibility_test.py
 create mode 100644 requirements.txt

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..dcae40a
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+.idea
+test.json
+venv
\ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..f58990b
--- /dev/null
+++ b/README.md
@@ -0,0 +1,41 @@
+# compatibility-test-suite-for-redis
+
+compatibility-test-suite-for-redis is used to test whether your redis-like database is compatible with Redis versions (such as
+6.0, 7.0, etc.)
+
+# Install
+
+requires `Python 3.7` or later.
+
+```
+pip3 install -r requirements.txt
+```
+
+# How to use
+
+```
+optional arguments:
+  -h, --help            show this help message and exit
+  --host HOST           the redis host
+  --port PORT           the redis port
+  --password PASSWORD   the redis password
+  --testfile TESTFILE   the redis compatibility test cases
+  --specific-version {1.0.0,2.8.0,4.0.0,5.0.0,6.0.0,6.2.0,7.0.0}
+                        the redis version
+  --show-failed         show details of failed tests
+  --cluster             server is a node of the Redis cluster
+  --ssl                 open ssl connection
+```
+
+Examples:  
+Test whether host:port is compatible with redis 6.2.0 and display failure case: 
+```
+$ python3 redis_compatibility_test.py -h host -p port --testfile cts.json --specific-version 6.2.0 --show-failed
+test: pexpiretime command version skipped
+test: persist command passed
+...
+test: set command passed
+-------- The result of tests --------
+version: 6.2.0, total tests: 17, passed: 17, rate: 100.0%
+```
+More examples are shown `python3 redis_compatibility_test.py -h`.
\ No newline at end of file
diff --git a/cts.json b/cts.json
new file mode 100644
index 0000000..c74ff9a
--- /dev/null
+++ b/cts.json
@@ -0,0 +1,496 @@
+[
+  {
+    "name": "del command",
+    "command": [
+      "set k v",
+      "del k"
+    ],
+    "result": [
+      "OK",
+      1
+    ],
+    "since": "1.0.0"
+  },
+  {
+    "name": "unlink command",
+    "command": [
+      "set k v",
+      "unlink k"
+    ],
+    "result": [
+      "OK",
+      1
+    ],
+    "since": "4.0.0"
+  },
+  {
+    "name": "rename command",
+    "command": [
+      "set k v",
+      "rename k kk"
+    ],
+    "result": [
+      "OK",
+      "OK"
+    ],
+    "since": "1.0.0"
+  },
+  {
+    "name": "renamenx command",
+    "command": [
+      "set k v",
+      "renamenx k kk"
+    ],
+    "result": [
+      "OK",
+      1
+    ],
+    "since": "1.0.0"
+  },
+  {
+    "name": "randomkey command",
+    "command": [
+      "set k v",
+      "randomkey"
+    ],
+    "result": [
+      "OK",
+      "k"
+    ],
+    "since": "1.0.0"
+  },
+  {
+    "name": "exists command",
+    "command": [
+      "set k v",
+      "exists k"
+    ],
+    "result": [
+      "OK",
+      1
+    ],
+    "since": "1.0.0"
+  },
+  {
+    "name": "ttl command",
+    "command": [
+      "ttl non-exists"
+    ],
+    "result": [
+      -2
+    ],
+    "since": "1.0.0"
+  },
+  {
+    "name": "pttl command",
+    "command": [
+      "pttl non-exists"
+    ],
+    "result": [
+      -2
+    ],
+    "since": "1.0.0"
+  },
+  {
+    "name": "expire command",
+    "command": [
+      "expire non-exists 10"
+    ],
+    "result": [
+      0
+    ],
+    "since": "1.0.0"
+  },
+  {
+    "name": "expire with nx/xx",
+    "command": [
+      "set k v",
+      "expire k 10 NX",
+      "expire k 10 XX"
+    ],
+    "result": [
+      "OK",
+      1,
+      1
+    ],
+    "since": "7.0.0"
+  },
+  {
+    "name": "expire with gt/lt",
+    "command": [
+      "set k v",
+      "expire k 10 lt",
+      "expire k 20 gt"
+    ],
+    "result": [
+      "OK",
+      1,
+      1
+    ],
+    "since": "7.0.0"
+  },
+  {
+    "name": "expireat command",
+    "command": [
+      "expireat non-exists 10"
+    ],
+    "result": [
+      0
+    ],
+    "since": "1.2.0"
+  },
+  {
+    "name": "expireat with nx/xx",
+    "command": [
+      "set k v",
+      "expireat k 9999999998 NX",
+      "expireat k 9999999999 XX"
+    ],
+    "result": [
+      "OK",
+      1,
+      1
+    ],
+    "since": "7.0.0"
+  },
+  {
+    "name": "expireat with gt/lt",
+    "command": [
+      "set k v",
+      "expireat k 9999999998 lt",
+      "expireat k 9999999999 gt"
+    ],
+    "result": [
+      "OK",
+      1,
+      1
+    ],
+    "since": "7.0.0"
+  },
+  {
+    "name": "pexpire command",
+    "command": [
+      "pexpire non-exists 10"
+    ],
+    "result": [
+      0
+    ],
+    "since": "2.6.0"
+  },
+  {
+    "name": "pexpire with nx/xx",
+    "command": [
+      "set k v",
+      "pexpire k 1000 NX",
+      "pexpire k 1000 XX"
+    ],
+    "result": [
+      "OK",
+      1,
+      1
+    ],
+    "since": "7.0.0"
+  },
+  {
+    "name": "pexpire with gt/lt",
+    "command": [
+      "set k v",
+      "pexpire k 1000 lt",
+      "pexpire k 2000 gt"
+    ],
+    "result": [
+      "OK",
+      1,
+      1
+    ],
+    "since": "7.0.0"
+  },
+  {
+    "name": "pexpireat command",
+    "command": [
+      "pexpireat non-exists 10"
+    ],
+    "result": [
+      0
+    ],
+    "since": "2.6.0"
+  },
+  {
+    "name": "pexpireat with nx/xx",
+    "command": [
+      "set k v",
+      "pexpireat k 9999999999998 NX",
+      "pexpireat k 9999999999999 XX"
+    ],
+    "result": [
+      "OK",
+      1,
+      1
+    ],
+    "since": "7.0.0"
+  },
+  {
+    "name": "pexpireat with gt/lt",
+    "command": [
+      "set k v",
+      "pexpireat k 9999999999998 lt",
+      "pexpireat k 9999999999999 gt"
+    ],
+    "result": [
+      "OK",
+      1,
+      1
+    ],
+    "since": "7.0.0"
+  },
+  {
+    "name": "expiretime command",
+    "command": [
+      "expiretime non-exists"
+    ],
+    "result": [
+      -2
+    ],
+    "since": "7.0.0"
+  },
+  {
+    "name": "pexpiretime command",
+    "command": [
+      "pexpiretime non-exists"
+    ],
+    "result": [
+      -2
+    ],
+    "since": "7.0.0"
+  },
+  {
+    "name": "persist command",
+    "command": [
+      "persist non-exists"
+    ],
+    "result": [
+      0
+    ],
+    "since": "2.2.0"
+  },
+  {
+    "name": "dump command",
+    "command": [
+      "dump not-exists"
+    ],
+    "result": [
+      null
+    ],
+    "since": "2.6.0"
+  },
+  {
+    "name": "touch command",
+    "command": [
+      "touch not-exists"
+    ],
+    "result": [
+      0
+    ],
+    "since": "3.2.1"
+  },
+  {
+    "name": "restore command",
+    "command": [
+      "restore k 0 \\x00\\x01v\\x06\\x00\\a\\xe5\\xa62\\xecm\\xb6]"
+    ],
+    "result": [
+      "OK"
+    ],
+    "since": "2.6.0",
+    "command_binary": true
+  },
+  {
+    "name": "restore with replace",
+    "command": [
+      "set k v",
+      "restore k 0 \\x00\\x01v\\x06\\x00\\a\\xe5\\xa62\\xecm\\xb6] REPLACE"
+    ],
+    "result": [
+      "OK",
+      "OK"
+    ],
+    "since": "3.0.0",
+    "command_binary": true
+  },
+  {
+    "name": "restore with absttl",
+    "command": [
+      "restore k 0 \\x00\\x01v\\x06\\x00\\a\\xe5\\xa62\\xecm\\xb6] ABSTTL",
+      "ttl k"
+    ],
+    "result": [
+      "OK",
+      -1
+    ],
+    "since": "5.0.0",
+    "command_binary": true
+  },
+  {
+    "name": "restore with idletime",
+    "command": [
+      "restore k 0 \\x00\\x01v\\x06\\x00\\a\\xe5\\xa62\\xecm\\xb6] IDLETIME 1000"
+    ],
+    "result": [
+      "OK"
+    ],
+    "since": "5.0.0",
+    "command_binary": true
+  },
+  {
+    "name": "scan command",
+    "command": [
+      "set k v",
+      "scan 0"
+    ],
+    "result": [
+      "OK",
+      [
+        "0",
+        [
+          "k"
+        ]
+      ]
+    ],
+    "since": "2.8.0"
+  },
+  {
+    "name": "scan with type",
+    "command": [
+      "geoadd geokey 0 0 value",
+      "scan 0 type zset"
+    ],
+    "result": [
+      1,
+      [
+        "0",
+        [
+          "geokey"
+        ]
+      ]
+    ],
+    "since": "6.0.0"
+  },
+  {
+    "name": "keys command",
+    "command": [
+      "mset firstname Jack lastname Stuntman age 35",
+      "keys a??"
+    ],
+    "result": [
+      "OK",
+      [
+        "age"
+      ]
+    ],
+    "since": "1.0.0"
+  },
+  {
+    "name": "move command",
+    "command": [
+      "set k v",
+      "move k 1"
+    ],
+    "result": [
+      "OK",
+      1
+    ],
+    "since": "1.0.0"
+  },
+  {
+    "name": "copy command",
+    "command": [
+      "set k v",
+      "copy k kk",
+      "get kk"
+    ],
+    "result": [
+      "OK",
+      1,
+      "v"
+    ],
+    "since": "6.2.0"
+  },
+  {
+    "name": "type command",
+    "command": [
+      "set k v",
+      "type k"
+    ],
+    "result": [
+      "OK",
+      "string"
+    ],
+    "since": "1.0.0"
+  },
+  {
+    "name": "wait command",
+    "command": [
+      "wait 0 1000"
+    ],
+    "result": [
+      0
+    ],
+    "since": "3.0.0"
+  },
+  {
+    "name": "sort command",
+    "command": [
+      "lpush list 5 3 4 1 2",
+      "sort list"
+    ],
+    "result": [
+      5,
+      [
+        "1",
+        "2",
+        "3",
+        "4",
+        "5"
+      ]
+    ],
+    "since": "1.0.0"
+  },
+  {
+    "name": "sort_ro command",
+    "command": [
+      "lpush list 5 3 4 1 2",
+      "sort_ro list"
+    ],
+    "result": [
+      5,
+      [
+        "1",
+        "2",
+        "3",
+        "4",
+        "5"
+      ]
+    ],
+    "since": "7.0.0"
+  },
+  {
+    "name": "migrate command",
+    "command": [
+      "MIGRATE 0.0.0.0 6379 k 0 0"
+    ],
+    "result": [
+      "NOKEY"
+    ],
+    "since": "2.6.0"
+  },
+  {
+    "name": "set command",
+    "command": [
+      "set k v"
+    ],
+    "result": [
+      "OK"
+    ],
+    "since": "1.0.0"
+  }
+]
\ No newline at end of file
diff --git a/redis_compatibility_test.py b/redis_compatibility_test.py
new file mode 100644
index 0000000..7a76893
--- /dev/null
+++ b/redis_compatibility_test.py
@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+import argparse
+import redis
+import json
+from dataclasses import dataclass
+from typing import List, Dict
+
+EXAMPLE = """
+Examples:
+
+Run tests without specifying a version
+    python3 redis_compatibility_test.py --testfile cts.json
+
+Run the test for compatibility with Redis 6.2.0
+    python3 redis_compatibility_test.py --testfile cts.json --specific-version 6.2.0
+
+Run the test whether it is compatible with Redis 6.2.0, and print the failure case
+    python3 redis_compatibility_test.py --testfile cts.json --specific-version 6.2.0 --show-failed
+"""
+
+
+@dataclass
+class FailedTest:
+    name: str
+    reason: object
+
+
+@dataclass
+class TestResult:
+    total: int
+    passed: int
+    failed: List[FailedTest]
+
+
+r: redis.Redis = None
+g_results: Dict[str, TestResult] = {}
+
+
+def report_result():
+    print(f"-------- The result of tests --------")
+    if args.specific_version:
+        total = passed = 0
+        failed: List[FailedTest] = []
+        for v, t in g_results.items():
+            total += t.total
+            passed += t.passed
+            failed.extend(t.failed)
+        print(f"version: {args.specific_version}, total tests: {total}, passed: {passed}, "
+              f"rate: {repr(passed / total * 100)}%")
+        if args.show_failed and len(failed) != 0:
+            print(f"This is failed tests for {args.specific_version}:")
+            print(failed)
+    else:
+        for v, t in sorted(g_results.items()):
+            print(f"version: {v}, total tests: {t.total}, passed: {t.passed}, "
+                  f"rate: {repr(t.passed / t.total * 100)}%")
+            if args.show_failed and len(t.failed) != 0:
+                print(f"This is failed tests for {v}:")
+                print(t.failed)
+
+
+def is_equal(left, right):
+    if type(left) is bytes and type(right) is str:
+        return left.decode() == right
+    elif type(left) is str and type(right) is bytes:
+        return left == right.decode()
+    else:
+        return left == right
+
+
+def test_passed(result):
+    print("passed")
+    result.total += 1
+    result.passed += 1
+
+
+def test_failed(result, name, e):
+    print("failed")
+    result.total += 1
+    result.failed.append(FailedTest(name=name, reason=e))
+
+
+def trans_result_to_bytes(result):
+    if type(result) is str:
+        return result.encode()
+    if type(result) is list:
+        for i in range(len(result)):
+            result[i] = trans_result_to_bytes(result[i])
+    if type(result) is map:
+        for k, v in result.items():
+            result[k.encode()] = trans_result_to_bytes(v)
+            del result[k]
+    return result
+
+
+def trans_cmd(test, cmd):
+    if 'command_binary' in test:
+        array = bytearray()
+        i = 0
+        while i < len(cmd):
+            if cmd[i] == '\\' and cmd[i + 1] == '\\':
+                array.append(92)
+                i += 2
+            elif cmd[i] == '\\' and cmd[i + 1] == '"':
+                array.append(34)
+                i += 2
+            elif cmd[i] == '\\' and cmd[i + 1] == 'n':
+                array.append(10)
+                i += 2
+            elif cmd[i] == '\\' and cmd[i + 1] == 'r':
+                array.append(13)
+                i += 2
+            elif cmd[i] == '\\' and cmd[i + 1] == 't':
+                array.append(9)
+                i += 2
+            elif cmd[i] == '\\' and cmd[i + 1] == 'a':
+                array.append(7)
+                i += 2
+            elif cmd[i] == '\\' and cmd[i + 1] == 'b':
+                array.append(8)
+                i += 2
+            elif cmd[i] == '\\' and cmd[i + 1] == 'x':
+                array.append(int(cmd[i + 2], 16) * 16 + int(cmd[i + 3], 16))
+                i += 4
+            else:
+                array.append(ord(cmd[i]))
+                i += 1
+        return bytes(array)
+    else:
+        return cmd
+
+
+def run_test(test):
+    name = test['name']
+    print(f"test: {name}", end=" ")
+    # if test need skipped
+    if 'skipped' in test:
+        print("skipped")
+        return
+
+    # high version test
+    since = test['since']
+    if args.specific_version and since > args.specific_version:
+        print("version skipped")
+        return
+    if since not in g_results:
+        g_results[since] = TestResult(total=0, passed=0, failed=[])
+
+    r.flushall()
+    command = test['command']
+    result = test['result']
+    trans_result_to_bytes(result)
+    try:
+        for idx, cmd in enumerate(command):
+            ret = r.execute_command(trans_cmd(test, cmd))
+            if result[idx] != ret:
+                test_failed(g_results[since], name, f"expected: {result[idx]}, result: {ret}")
+                return
+        test_passed(g_results[since])
+    except Exception as e:
+        test_failed(g_results[since], name, e)
+
+
+def run_compatibility_tests(filename):
+    with open(filename, "r") as f:
+        tests = f.read()
+    tests_array = json.loads(tests)
+    for test in tests_array:
+        run_test(test)
+
+
+def create_client(args):
+    global r
+    if args.cluster:
+        print(f"Connecting to {args.host}:{args.port} use cluster client")
+        r = redis.RedisCluster(host=args.host, port=args.port, password=args.password, ssl=args.ssl)
+        assert r.ping()
+    else:
+        print(f"Connecting to {args.host}:{args.port} use standalone client")
+        r = redis.Redis(host=args.host, port=args.port, password=args.password, ssl=args.ssl)
+        assert r.ping()
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(prog="redis_compatibility_test",
+                                     description="redis_compatibility_test is used to test whether your redis-like "
+                                                 "database is compatible with Redis versions (such as 6.0, 7.0, etc.)",
+                                     formatter_class=argparse.RawDescriptionHelpFormatter,
+                                     epilog=EXAMPLE)
+    parser.add_argument("--host", help="the redis host", default="127.0.0.1")
+    parser.add_argument("--port", help="the redis port", default=6379, type=int)
+    parser.add_argument("--password", help="the redis password", default="")
+    parser.add_argument("--testfile", help="the redis compatibility test cases", required=True)
+    parser.add_argument("--specific-version", dest="specific_version", help="the redis version",
+                        choices=['1.0.0', '2.8.0', '4.0.0', '5.0.0', '6.0.0', '6.2.0', '7.0.0'])
+    parser.add_argument("--show-failed", dest="show_failed", help="show details of failed tests", default=False,
+                        action="store_true")
+    parser.add_argument("--cluster", help="server is a node of the Redis cluster", default=False, action="store_true")
+    parser.add_argument("--ssl", help="open ssl connection", default=False, action="store_true")
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    create_client(args)
+    run_compatibility_tests(args.testfile)
+    report_result()
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..efab893
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+redis>=4.3.4