Tair
/
compatibility-test-suite-for-redis
mirror of https://github.com/tair-opensource/compatibility-test-suite-for-redis.git
3
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Add --genhtml option for test

This commit is contained in:
bodong.ybd 2023-08-09 21:04:18 +08:00
parent 37e78618b4
commit 7e1df3171d
5 changed files with 267 additions and 248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1,3 +1,4 @@
.idea
test.json
venv
venv
html

105
config.yaml Normal file
View File

@ -0,0 +1,105 @@
# config.yaml is used when you need to test the compatibility of multiple Redis-Like
# systems and generate html reports (--genhtml)
# html report directory
Dir: "html"
# specify test version
SpecificVersion:
- 4.0.0
- 5.0.0
- 6.0.0
- 7.0.0
# the host, port, password of all the db to be tested
Database:
Redis:
host: 127.0.0.1
port: 6379
password:
ssl: false
cluster: false
DragonflyDB:
host: 127.0.0.1
port: 6380
password:
ssl: false
cluster: false
Kvrocks:
host: 127.0.0.1
port: 6381
password:
ssl: false
cluster: false
KeyDB:
host: 127.0.0.1
port: 6382
password:
ssl: false
cluster: false
Pika:
host: 127.0.0.1
port: 6383
password:
ssl: false
cluster: false
Tair:
host:
port:
password:
ssl:
cluster: false
Kvstore:
host:
port:
password:
ssl:
cluster: false
ElastiCache:
host:
port:
password:
ssl:
cluster: false
MemoryDB:
host:
port:
password:
ssl:
cluster: false
AzureRedis:
host:
port:
password:
ssl:
cluster: false
GoogleMemoryStore:
host:
port:
password:
ssl:
cluster: false
TencentRedis:
host:
port:
password:
ssl:
cluster: false
GaussDBRedis:
host:
port:
password:
ssl:
cluster: false

243
cts.json
View File

@ -462,17 +462,6 @@
"since": "1.0.0"
},
{
"name": "wait command",
"command": [
"wait 0 1000"
],
"result": [
0
],
"since": "3.0.0"
},
{
"name": "sort command",
"command": [
@ -511,17 +500,6 @@
"since": "7.0.0"
},
{
"name": "migrate command",
"command": [
"MIGRATE 0.0.0.0 6379 k 0 0"
],
"result": [
"NOKEY"
],
"since": "2.6.0"
},
{
"name": "set command",
"command": [
@ -1327,7 +1305,8 @@
"1"
]
],
"since": "1.0.0"
"since": "1.0.0",
"sort_result": true
},
{
@ -1449,7 +1428,8 @@
"1"
]
],
"since": "1.0.0"
"since": "1.0.0",
"sort_result": true
},
{
@ -1624,7 +1604,8 @@
"1"
]
],
"since": "1.0.0"
"since": "1.0.0",
"sort_result": true
},
{
@ -3153,14 +3134,16 @@
{
"name": "incrbyfloat command",
"command": [
"set mykey 10.50",
"incrbyfloat mykey 0.1",
"get mykey"
"set mykey 0.5",
"incrbyfloat mykey 1.123",
"get mykey",
"del mykey"
],
"result": [
"OK",
"10.6",
"10.6"
"1.623",
"1.623",
1
],
"since": "2.6.0"
},
@ -3563,7 +3546,8 @@
"field1", "1"
]
],
"since": "2.0.0"
"since": "2.0.0",
"sort_result": true
},
{
@ -3720,7 +3704,8 @@
]
]
],
"since": "2.8.0"
"since": "2.8.0",
"sort_result": true
},
{
@ -3739,7 +3724,8 @@
]
]
],
"since": "2.8.0"
"since": "2.8.0",
"sort_result": true
},
{
@ -3770,7 +3756,8 @@
"field1", "1"
]
],
"since": "4.0.0"
"since": "4.0.0",
"sort_result": true
},
{
@ -4967,146 +4954,6 @@
"since": "7.0.0"
},
{
"name": "acl deluser command",
"command": [
"acl deluser nonexisting"
],
"result": [
0
],
"since": "6.0.0"
},
{
"name": "acl dryrun command",
"command": [
"acl dryrun default set foo bar"
],
"result": [
"OK"
],
"since": "7.0.0"
},
{
"name": "acl log command",
"command": [
"acl log"
],
"result": [
[]
],
"since": "6.0.0"
},
{
"name": "acl setuser command",
"command": [
"acl setuser test reset",
"acl deluser test"
],
"result": [
"OK",
1
],
"since": "6.0.0"
},
{
"name": "acl users command",
"command": [
"acl users"
],
"result": [
[
"default"
]
],
"since": "6.0.0"
},
{
"name": "acl whoami command",
"command": [
"acl whoami"
],
"result": [
"default"
],
"since": "6.0.0"
},
{
"name": "save & bgsave command",
"command": [
"save",
"bgsave"
],
"result": [
"True",
"True"
],
"since": "1.0.0"
},
{
"name": "bgrewriteaof command",
"command": [
"bgrewriteaof",
"config set appendfsync no"
],
"result": [
"True",
"OK"
],
"since": "1.0.0"
},
{
"name": "config get command",
"command": [
"config get 1"
],
"result": [
[]
],
"since": "2.0.0"
},
{
"name": "config get with multiple arguments",
"command": [
"config get 1 2"
],
"result": [
[]
],
"since": "7.0.0"
},
{
"name": "config resetstat command",
"command": [
"config resetstat"
],
"result": [
"True"
],
"since": "2.0.0"
},
{
"name": "config set command",
"command": [
"config set appendfsync no"
],
"result": [
"OK"
],
"since": "2.0.0"
},
{
"name": "dbsize command",
"command": [
@ -5184,54 +5031,6 @@
"since": "6.2.0"
},
{
"name": "module list command",
"command": [
"module list"
],
"result": [
[]
],
"since": "4.0.0"
},
{
"name": "replicaof command",
"command": [
"replicaof no one"
],
"result": [
"OK"
],
"since": "5.0.0"
},
{
"name": "role command",
"command": [
"role"
],
"result": [
[
"master",
0,
[]
]
],
"since": "2.8.12"
},
{
"name": "slaveof command",
"command": [
"slaveof no one"
],
"result": [
"OK"
],
"since": "1.0.0"
},
{
"name": "swapdb command",
"command": [

163
redis_compatibility_test.py
View File

@ -1,7 +1,12 @@
#!/usr/bin/env python3
import argparse
import os
import re
import redis
import json
import yaml
import shutil
from dataclasses import dataclass
from typing import List, Dict
@ -34,10 +39,11 @@ class TestResult:
r: redis.Redis = None
g_results: Dict[str, TestResult] = {}
logfile = None
def report_result():
print(f"-------- The result of tests --------")
print(f"-------- The result of tests --------", file=logfile)
if args.specific_version:
total = passed = 0
failed: List[FailedTest] = []
@ -45,20 +51,21 @@ def report_result():
total += t.total
passed += t.passed
failed.extend(t.failed)
print(f"version: {args.specific_version}, total tests: {total}, passed: {passed}, "
f"rate: {repr(passed / total * 100)}%")
rate = passed / total * 100
print(f"Summary: version: {args.specific_version}, total tests: {total}, passed: {passed}, "
f"rate: {rate:.2f}%", file=logfile)
if args.show_failed and len(failed) != 0:
print(f"This is failed tests for {args.specific_version}:")
print('\n'.join(str(fail) for fail in failed))
exit(-1)
print(f"This is failed tests for {args.specific_version}:", file=logfile)
print('\n'.join(str(fail) for fail in failed), file=logfile)
else:
for v, t in sorted(g_results.items()):
print(f"version: {v}, total tests: {t.total}, passed: {t.passed}, "
f"rate: {repr(t.passed / t.total * 100)}%")
rate = t.passed / t.total * 100
print(f"Summary: version: {v}, total tests: {t.total}, passed: {t.passed}, "
f"rate: {rate:.2f}%", file=logfile)
for v, t in sorted(g_results.items()):
if args.show_failed and len(t.failed) != 0:
print(f"This is failed tests for {v}:")
print('\n'.join(str(fail) for fail in t.failed))
print(f"This is failed tests for {v}:", file=logfile)
print('\n'.join(str(fail) for fail in t.failed), file=logfile)
def is_equal(left, right):
@ -71,13 +78,13 @@ def is_equal(left, right):
def test_passed(result):
print("passed")
print("passed", file=logfile)
result.total += 1
result.passed += 1
def test_failed(result, name, e):
print("failed")
print("failed", file=logfile)
result.total += 1
result.failed.append(FailedTest(name=name, reason=e))
@ -151,18 +158,28 @@ def trans_cmd(test, cmd):
return cmd
def sort_nested_list(result):
nested = False
for ele in result:
if isinstance(ele, list):
ele.sort()
nested = True
if not nested:
result.sort()
def run_test(test):
name = test['name']
print(f"test: {name}", end=" ")
print(f"test: {name}", end=" ", file=logfile)
# if test need skipped
if 'skipped' in test:
print("skipped")
print("skipped", file=logfile)
return
# high version test
since = test['since']
if args.specific_version and since > args.specific_version:
print("version skipped")
print("version skipped", file=logfile)
return
if since not in g_results:
g_results[since] = TestResult(total=0, passed=0, failed=[])
@ -178,6 +195,9 @@ def run_test(test):
ret = trans_result_to_bytes(r.execute_command(*trans_cmd(test, cmd)))
else:
ret = trans_result_to_bytes(r.execute_command(trans_cmd(test, cmd)))
if 'sort_result' in test and isinstance(result[idx], list):
sort_nested_list(ret)
sort_nested_list(result[idx])
if result[idx] != ret:
test_failed(g_results[since], name, f"expected: {result[idx]}, result: {ret}")
return
@ -191,18 +211,107 @@ def run_compatibility_tests(filename):
tests = f.read()
tests_array = json.loads(tests)
for test in tests_array:
run_test(test)
try:
run_test(test)
except Exception as e:
print(f"run test error {e}", file=logfile)
continue
def create_client(args):
def generate_html_report(logdir, configs):
filepath = f"{logdir}/index.html"
html = open(filepath, "w")
html.write("This page is automatically generated by <a href=\"https://github.com/tair-opensource/"
"compatibility-test-suite-for-redis\">compatibility-test-suite-for-redis</a>"
" to show the compatibility of the following Redis-Like systems and different versions of Redis.<br><br>")
html.write("<table>")
# generate header
html.write("<thead>")
html.write("<tr>")
html.write("<th>Product / Redis Version</th>")
for version in configs['SpecificVersion']:
html.write(f"<th>{version}</th>")
html.write("</tr>")
html.write("</thead>")
# generate body
html.write("<tbody>")
for config in configs['Database']:
html.write("<tr>")
html.write(f"<td>{config}</td>")
for version in configs['SpecificVersion']:
filepath = f"{logdir}/{config}-{version}.html"
if not os.path.exists(filepath):
html.write(f"<td>-</td>")
continue
with open(filepath, 'r') as f:
s = f.read()
match = re.search(r"rate: (\d+\.\d+)%", s)
assert match
rate = match.group(1)
color = "#40de5a"
if eval(rate) < 80:
color = "#f05654"
elif eval(rate) < 100:
color = "#ffa400"
html.write(f"<td style=\"background:{color}\">{rate}% <a href=\"{config}-{version}.html\">detail</a></td>")
html.write("</tr>")
html.write("</tbody>")
html.write("</table>")
html.write("<style>table {border-collapse: collapse;} th, td {border: 1px solid black; padding: 8px;}</style>")
html.close()
def run_test_by_configfile():
global logfile
try:
with open('config.yaml', 'r') as f:
configs = yaml.load(f, Loader=yaml.FullLoader)
except FileNotFoundError as e:
print(f"error {e}")
exit(-1)
logdir = configs['Dir']
if os.path.exists(logdir):
print(f"directory {logdir} already exists, will be deleted and renew.")
shutil.rmtree(logdir)
os.makedirs(logdir)
for config in configs['Database']:
for version in configs['SpecificVersion']:
print(f"start test {config} for version {version}")
try:
create_client(configs['Database'][config]['host'],
configs['Database'][config]['port'],
configs['Database'][config]['password'],
configs['Database'][config]['ssl'],
configs['Database'][config]['cluster'])
except Exception as e:
print(f"connect to {configs['Database'][config]['host']}:{configs['Database'][config]['port']} "
f"fail, skip this test, error {e}")
break
filepath = f"{logdir}/{config}-{version}.html"
logfile = open(filepath, "w")
args.specific_version = version
args.show_failed = True
g_results.clear()
print("<pre>", file=logfile)
run_compatibility_tests(args.testfile)
report_result()
print("</pre>", file=logfile)
logfile.close()
logfile = None
# now we generate index.html
generate_html_report(logdir, configs)
def create_client(host, port, password, ssl, cluster):
global r
if args.cluster:
print(f"Connecting to {args.host}:{args.port} use cluster client")
r = redis.RedisCluster(host=args.host, port=args.port, password=args.password, ssl=args.ssl)
if cluster:
print(f"connecting to {host}:{port} use cluster client", file=logfile)
r = redis.RedisCluster(host=host, port=port, password=password, ssl=ssl)
assert r.ping()
else:
print(f"Connecting to {args.host}:{args.port} use standalone client")
r = redis.Redis(host=args.host, port=args.port, password=args.password, ssl=args.ssl)
print(f"connecting to {host}:{port} use standalone client", file=logfile)
r = redis.Redis(host=host, port=port, password=password, ssl=ssl)
assert r.ping()
@ -222,11 +331,15 @@ def parse_args():
action="store_true")
parser.add_argument("--cluster", help="server is a node of the Redis cluster", default=False, action="store_true")
parser.add_argument("--ssl", help="open ssl connection", default=False, action="store_true")
parser.add_argument("--genhtml", help="generate test report in html format", default=False, action="store_true")
return parser.parse_args()
if __name__ == '__main__':
args = parse_args()
create_client(args)
run_compatibility_tests(args.testfile)
report_result()
if args.genhtml:
run_test_by_configfile()
else:
create_client(args.host, args.port, args.password, args.ssl, args.cluster)
run_compatibility_tests(args.testfile)
report_result()

1
requirements.txt
View File

@ -1 +1,2 @@
redis>=4.3.4
pyyaml>=6.0