dnrops
/
binlog2sql
mirror of https://github.com/danfengcao/binlog2sql.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

let options fit user habit

This commit is contained in:
danfengcao 2016-12-16 18:12:47 +08:00
parent 3747167939
commit b3dd1a8bcf
4 changed files with 30 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
README.md
View File

@ -40,13 +40,14 @@ shell> pip install -r requirements.txt
binlog-format = row
### 需要给client端赋予的最小权限集合：
select, super/replication client, replication slave
**权限说明:**
select, super/replication client, replication slave
select：需要读取server端的information_schema.COLUMNS表，获取表结构的元信息，拼接成可视化的sql语句
super/replication client：两个权限都可以，需要执行'SHOW MASTER STATUS', 获取server端的binlog列表
replication slave：通过BINLOG_DUMP协议获取binlog内容的权限
**权限说明**
* select：需要读取server端的information_schema.COLUMNS表，获取表结构的元信息，拼接成可视化的sql语句
* super/replication client：两个权限都可以，需要执行'SHOW MASTER STATUS', 获取server端的binlog列表
* replication slave：通过BINLOG_DUMP协议获取binlog内容的权限
###基本用法
@ -80,9 +81,9 @@ UPDATE `test`.`test3` SET `addtime`='2016-12-10 13:03:22', `data`='中文', `id`
--stop-never 持续同步binlog。可选。不加则同步至执行命令时最新的binlog位置。
--popPk 对INSERT语句去除主键。可选。
-K, --no-primary-key 对INSERT语句去除主键。可选。
-B, --flashback 生成回滚语句。可选。与stop-never或popPk不能同时添加。
-B, --flashback 生成回滚语句。可选。与stop-never或no-primary-key不能同时添加。
**解析范围控制**
@ -158,7 +159,7 @@ DELETE FROM `test`.`tbl` WHERE `addtime`='2016-12-10 00:04:48' AND `id`=2 AND `n
DELETE FROM `test`.`tbl` WHERE `addtime`='2016-12-13 20:25:00' AND `id`=3 AND `name`='小孙' LIMIT 1; #start 728 end 938 time 2016-12-13 20:28:05
DELETE FROM `test`.`tbl` WHERE `addtime`='2016-12-12 00:00:00' AND `id`=4 AND `name`='小李' LIMIT 1; #start 728 end 938 time 2016-12-13 20:28:05
```
3. 我们得到了误操作sql的准确位置在728-938之间，再根据位置进一步过滤，使用flashback模式生成回滚sql，检查回滚sql是否正确
```bash
@ -169,7 +170,7 @@ INSERT INTO `test`.`tbl`(`addtime`, `id`, `name`) VALUES ('2016-12-13 20:25:00',
INSERT INTO `test`.`tbl`(`addtime`, `id`, `name`) VALUES ('2016-12-10 00:04:48', 2, '小钱'); #start 728 end 938 time 2016-12-13 20:28:05
INSERT INTO `test`.`tbl`(`addtime`, `id`, `name`) VALUES ('2016-12-10 00:04:33', 1, '小赵'); #start 728 end 938 time 2016-12-13 20:28:05
```
3. 确认回滚sql正确，执行回滚语句。登录mysql确认，数据回滚成功。
```bash
@ -194,7 +195,7 @@ mysql> select * from tbl;
###优点（对比mysqlbinlog）
* 纯Python开发，安装与使用都很简单
* 自带flashback、popPk解析模式，无需再装补丁
* 自带flashback、no-primary-key解析模式，无需再装补丁
* 解析为标准SQL，方便理解、调试
* 代码容易改造，可以支持更多个性化解析

10
binlog2sql/binlog2sql.py
View File

@ -15,7 +15,7 @@ from binlog2sql_util import command_line_args, concat_sql_from_binlogevent, crea
class Binlog2sql(object):
def __init__(self, connectionSettings, startFile=None, startPos=None, endFile=None, endPos=None, startTime=None,
stopTime=None, only_schemas=None, only_tables=None, popPk=False, flashback=False, stopnever=False):
stopTime=None, only_schemas=None, only_tables=None, nopk=False, flashback=False, stopnever=False):
'''
connectionSettings: {'host': 127.0.0.1, 'port': 3306, 'user': slave, 'passwd': slave}
'''
@ -32,7 +32,7 @@ class Binlog2sql(object):
self.only_schemas = only_schemas if only_schemas else None
self.only_tables = only_tables if only_tables else None
self.popPk, self.flashback, self.stopnever = (popPk, flashback, stopnever)
self.nopk, self.flashback, self.stopnever = (nopk, flashback, stopnever)
self.binlogList = []
self.connection = pymysql.connect(**self.connectionSettings)
@ -82,12 +82,12 @@ class Binlog2sql(object):
eStartPos = lastPos
if isinstance(binlogevent, QueryEvent):
sql = concat_sql_from_binlogevent(cursor=cur, binlogevent=binlogevent, flashback=self.flashback, popPk=self.popPk)
sql = concat_sql_from_binlogevent(cursor=cur, binlogevent=binlogevent, flashback=self.flashback, nopk=self.nopk)
if sql:
print sql
elif type(binlogevent) in (WriteRowsEvent, UpdateRowsEvent, DeleteRowsEvent):
for row in binlogevent.rows:
sql = concat_sql_from_binlogevent(cursor=cur, binlogevent=binlogevent, row=row , flashback=self.flashback, popPk=self.popPk, eStartPos=eStartPos)
sql = concat_sql_from_binlogevent(cursor=cur, binlogevent=binlogevent, row=row , flashback=self.flashback, nopk=self.nopk, eStartPos=eStartPos)
if self.flashback:
ftmp.write(sql + '\n')
else:
@ -120,5 +120,5 @@ if __name__ == '__main__':
binlog2sql = Binlog2sql(connectionSettings=connectionSettings, startFile=args.startFile,
startPos=args.startPos, endFile=args.endFile, endPos=args.endPos,
startTime=args.startTime, stopTime=args.stopTime, only_schemas=args.databases,
only_tables=args.tables, popPk=args.popPk, flashback=args.flashback, stopnever=args.stopnever)
only_tables=args.tables, nopk=args.nopk, flashback=args.flashback, stopnever=args.stopnever)
binlog2sql.process_binlog()

26
binlog2sql/binlog2sql_util.py
View File

@ -47,12 +47,12 @@ def command_line_parser():
range = parser.add_argument_group('range filter')
range.add_argument('--start-file', dest='startFile', type=str,
help='Start binlog file to be parsed')
range.add_argument('--start-pos', dest='startPos', type=int,
help='start position of the --start-file', default=4)
range.add_argument('--end-file', dest='endFile', type=str,
help="End binlog file to be parsed. default: '--start-file'", default='')
range.add_argument('--end-pos', dest='endPos', type=int,
help="stop position of --end-file. default: end position of '--end-file'", default=0)
range.add_argument('--start-position', '--start-pos', dest='startPos', type=int,
help='Start position of the --start-file', default=4)
range.add_argument('--stop-file', '--end-file', dest='endFile', type=str,
help="Stop binlog file to be parsed. default: '--start-file'", default='')
range.add_argument('--stop-position', '--end-pos', dest='endPos', type=int,
help="Stop position of --stop-file. default: latest position of '--stop-file'", default=0)
range.add_argument('--start-datetime', dest='startTime', type=str,
help="Start reading the binlog at first event having a datetime equal or posterior to the argument; the argument must be a date and time in the local time zone, in any format accepted by the MySQL server for DATETIME and TIMESTAMP types, for example: 2004-12-25 11:25:56 (you should probably use quotes for your shell to set it properly).", default='')
range.add_argument('--stop-datetime', dest='stopTime', type=str,
@ -69,7 +69,7 @@ def command_line_parser():
help='tables you want to process', default='')
exclusive = parser.add_mutually_exclusive_group()
exclusive.add_argument('--popPk', dest='popPk', action='store_true',
exclusive.add_argument('-K', '--no-primary-key', dest='nopk', action='store_true',
help='Generate insert sql without primary key if exists', default=False)
exclusive.add_argument('-B', '--flashback', dest='flashback', action='store_true',
help='Flashback data to start_postition of start_file', default=False)
@ -83,8 +83,8 @@ def command_line_args():
sys.exit(1)
if args.flashback and args.stopnever:
raise ValueError('only one of flashback or stop-never can be True')
if args.flashback and args.popPk:
raise ValueError('only one of flashback or popPk can be True')
if args.flashback and args.nopk:
raise ValueError('only one of flashback or nopk can be True')
if (args.startTime and not is_valid_datetime(args.startTime)) or (args.stopTime and not is_valid_datetime(args.stopTime)):
raise ValueError('Incorrect date and time argument')
return args
@ -104,9 +104,9 @@ def fix_object(value):
else:
return value
def concat_sql_from_binlogevent(cursor, binlogevent, row=None, eStartPos=None, flashback=False, popPk=False):
if flashback and popPk:
raise ValueError('only one of flashback or popPk can be True')
def concat_sql_from_binlogevent(cursor, binlogevent, row=None, eStartPos=None, flashback=False, nopk=False):
if flashback and nopk:
raise ValueError('only one of flashback or nopk can be True')
if type(binlogevent) not in (WriteRowsEvent, UpdateRowsEvent, DeleteRowsEvent, QueryEvent):
raise ValueError('binlogevent must be WriteRowsEvent, UpdateRowsEvent, DeleteRowsEvent or QueryEvent')
@ -133,7 +133,7 @@ def concat_sql_from_binlogevent(cursor, binlogevent, row=None, eStartPos=None, f
sql = cursor.mogrify(template, map(fix_object, row['before_values'].values()+row['after_values'].values()))
else:
if isinstance(binlogevent, WriteRowsEvent):
if popPk:
if nopk:
tableInfo = (binlogevent.table_map)[binlogevent.table_id]
if tableInfo.primary_key:
row['values'].pop(tableInfo.primary_key)

2
example/FixOldMasterExtraData.md
View File

@ -41,7 +41,7 @@ MySQL主从切换后如何进行数据修复
我们使用了纯Python开发的工具[binlog2sql](https://github.com/danfengcao/binlog2sql)来操作，执行命令如下
```bash
$ python binlog2sql.py --popPk -h10.1.1.1 -P3306 -uadmin -p'admin' --start-file='mysql-bin.000040' --start-pos=125466 --end-file='mysql-bin.000041' > oldMaster.sql
$ python binlog2sql.py --no-primary-key -h10.1.1.1 -P3306 -uadmin -p'admin' --start-file='mysql-bin.000040' --start-pos=125466 --end-file='mysql-bin.000041' > oldMaster.sql
$ python binlog2sql.py --flashback -h10.1.1.1 -P3306 -uadmin -p'admin' --start-file='mysql-bin.mysql-bin.000040' --start-pos=125466 --end-file='mysql-bin.000041' | mysql -h10.1.1.1 -P3306 -uadmin -p'admin'