diff --git a/README.md b/README.md index 8faee14..6ec69f4 100644 --- a/README.md +++ b/README.md @@ -40,13 +40,14 @@ shell> pip install -r requirements.txt binlog-format = row ### 需要给client端赋予的最小权限集合: -select, super/replication client, replication slave -**权限说明:** + select, super/replication client, replication slave -select:需要读取server端的information_schema.COLUMNS表,获取表结构的元信息,拼接成可视化的sql语句 -super/replication client:两个权限都可以,需要执行'SHOW MASTER STATUS', 获取server端的binlog列表 -replication slave:通过BINLOG_DUMP协议获取binlog内容的权限 +**权限说明** + +* select:需要读取server端的information_schema.COLUMNS表,获取表结构的元信息,拼接成可视化的sql语句 +* super/replication client:两个权限都可以,需要执行'SHOW MASTER STATUS', 获取server端的binlog列表 +* replication slave:通过BINLOG_DUMP协议获取binlog内容的权限 ###基本用法 @@ -80,9 +81,9 @@ UPDATE `test`.`test3` SET `addtime`='2016-12-10 13:03:22', `data`='中文', `id` --stop-never 持续同步binlog。可选。不加则同步至执行命令时最新的binlog位置。 ---popPk 对INSERT语句去除主键。可选。 +-K, --no-primary-key 对INSERT语句去除主键。可选。 --B, --flashback 生成回滚语句。可选。与stop-never或popPk不能同时添加。 +-B, --flashback 生成回滚语句。可选。与stop-never或no-primary-key不能同时添加。 **解析范围控制** @@ -158,7 +159,7 @@ DELETE FROM `test`.`tbl` WHERE `addtime`='2016-12-10 00:04:48' AND `id`=2 AND `n DELETE FROM `test`.`tbl` WHERE `addtime`='2016-12-13 20:25:00' AND `id`=3 AND `name`='小孙' LIMIT 1; #start 728 end 938 time 2016-12-13 20:28:05 DELETE FROM `test`.`tbl` WHERE `addtime`='2016-12-12 00:00:00' AND `id`=4 AND `name`='小李' LIMIT 1; #start 728 end 938 time 2016-12-13 20:28:05 ``` - + 3. 我们得到了误操作sql的准确位置在728-938之间,再根据位置进一步过滤,使用flashback模式生成回滚sql,检查回滚sql是否正确 ```bash @@ -169,7 +170,7 @@ INSERT INTO `test`.`tbl`(`addtime`, `id`, `name`) VALUES ('2016-12-13 20:25:00', INSERT INTO `test`.`tbl`(`addtime`, `id`, `name`) VALUES ('2016-12-10 00:04:48', 2, '小钱'); #start 728 end 938 time 2016-12-13 20:28:05 INSERT INTO `test`.`tbl`(`addtime`, `id`, `name`) VALUES ('2016-12-10 00:04:33', 1, '小赵'); #start 728 end 938 time 2016-12-13 20:28:05 ``` - + 3. 确认回滚sql正确,执行回滚语句。登录mysql确认,数据回滚成功。 ```bash @@ -194,7 +195,7 @@ mysql> select * from tbl; ###优点(对比mysqlbinlog) * 纯Python开发,安装与使用都很简单 -* 自带flashback、popPk解析模式,无需再装补丁 +* 自带flashback、no-primary-key解析模式,无需再装补丁 * 解析为标准SQL,方便理解、调试 * 代码容易改造,可以支持更多个性化解析 diff --git a/binlog2sql/binlog2sql.py b/binlog2sql/binlog2sql.py index 9c13dd4..f34dc71 100755 --- a/binlog2sql/binlog2sql.py +++ b/binlog2sql/binlog2sql.py @@ -15,7 +15,7 @@ from binlog2sql_util import command_line_args, concat_sql_from_binlogevent, crea class Binlog2sql(object): def __init__(self, connectionSettings, startFile=None, startPos=None, endFile=None, endPos=None, startTime=None, - stopTime=None, only_schemas=None, only_tables=None, popPk=False, flashback=False, stopnever=False): + stopTime=None, only_schemas=None, only_tables=None, nopk=False, flashback=False, stopnever=False): ''' connectionSettings: {'host': 127.0.0.1, 'port': 3306, 'user': slave, 'passwd': slave} ''' @@ -32,7 +32,7 @@ class Binlog2sql(object): self.only_schemas = only_schemas if only_schemas else None self.only_tables = only_tables if only_tables else None - self.popPk, self.flashback, self.stopnever = (popPk, flashback, stopnever) + self.nopk, self.flashback, self.stopnever = (nopk, flashback, stopnever) self.binlogList = [] self.connection = pymysql.connect(**self.connectionSettings) @@ -82,12 +82,12 @@ class Binlog2sql(object): eStartPos = lastPos if isinstance(binlogevent, QueryEvent): - sql = concat_sql_from_binlogevent(cursor=cur, binlogevent=binlogevent, flashback=self.flashback, popPk=self.popPk) + sql = concat_sql_from_binlogevent(cursor=cur, binlogevent=binlogevent, flashback=self.flashback, nopk=self.nopk) if sql: print sql elif type(binlogevent) in (WriteRowsEvent, UpdateRowsEvent, DeleteRowsEvent): for row in binlogevent.rows: - sql = concat_sql_from_binlogevent(cursor=cur, binlogevent=binlogevent, row=row , flashback=self.flashback, popPk=self.popPk, eStartPos=eStartPos) + sql = concat_sql_from_binlogevent(cursor=cur, binlogevent=binlogevent, row=row , flashback=self.flashback, nopk=self.nopk, eStartPos=eStartPos) if self.flashback: ftmp.write(sql + '\n') else: @@ -120,5 +120,5 @@ if __name__ == '__main__': binlog2sql = Binlog2sql(connectionSettings=connectionSettings, startFile=args.startFile, startPos=args.startPos, endFile=args.endFile, endPos=args.endPos, startTime=args.startTime, stopTime=args.stopTime, only_schemas=args.databases, - only_tables=args.tables, popPk=args.popPk, flashback=args.flashback, stopnever=args.stopnever) + only_tables=args.tables, nopk=args.nopk, flashback=args.flashback, stopnever=args.stopnever) binlog2sql.process_binlog() diff --git a/binlog2sql/binlog2sql_util.py b/binlog2sql/binlog2sql_util.py index f56335d..b0419ba 100755 --- a/binlog2sql/binlog2sql_util.py +++ b/binlog2sql/binlog2sql_util.py @@ -47,12 +47,12 @@ def command_line_parser(): range = parser.add_argument_group('range filter') range.add_argument('--start-file', dest='startFile', type=str, help='Start binlog file to be parsed') - range.add_argument('--start-pos', dest='startPos', type=int, - help='start position of the --start-file', default=4) - range.add_argument('--end-file', dest='endFile', type=str, - help="End binlog file to be parsed. default: '--start-file'", default='') - range.add_argument('--end-pos', dest='endPos', type=int, - help="stop position of --end-file. default: end position of '--end-file'", default=0) + range.add_argument('--start-position', '--start-pos', dest='startPos', type=int, + help='Start position of the --start-file', default=4) + range.add_argument('--stop-file', '--end-file', dest='endFile', type=str, + help="Stop binlog file to be parsed. default: '--start-file'", default='') + range.add_argument('--stop-position', '--end-pos', dest='endPos', type=int, + help="Stop position of --stop-file. default: latest position of '--stop-file'", default=0) range.add_argument('--start-datetime', dest='startTime', type=str, help="Start reading the binlog at first event having a datetime equal or posterior to the argument; the argument must be a date and time in the local time zone, in any format accepted by the MySQL server for DATETIME and TIMESTAMP types, for example: 2004-12-25 11:25:56 (you should probably use quotes for your shell to set it properly).", default='') range.add_argument('--stop-datetime', dest='stopTime', type=str, @@ -69,7 +69,7 @@ def command_line_parser(): help='tables you want to process', default='') exclusive = parser.add_mutually_exclusive_group() - exclusive.add_argument('--popPk', dest='popPk', action='store_true', + exclusive.add_argument('-K', '--no-primary-key', dest='nopk', action='store_true', help='Generate insert sql without primary key if exists', default=False) exclusive.add_argument('-B', '--flashback', dest='flashback', action='store_true', help='Flashback data to start_postition of start_file', default=False) @@ -83,8 +83,8 @@ def command_line_args(): sys.exit(1) if args.flashback and args.stopnever: raise ValueError('only one of flashback or stop-never can be True') - if args.flashback and args.popPk: - raise ValueError('only one of flashback or popPk can be True') + if args.flashback and args.nopk: + raise ValueError('only one of flashback or nopk can be True') if (args.startTime and not is_valid_datetime(args.startTime)) or (args.stopTime and not is_valid_datetime(args.stopTime)): raise ValueError('Incorrect date and time argument') return args @@ -104,9 +104,9 @@ def fix_object(value): else: return value -def concat_sql_from_binlogevent(cursor, binlogevent, row=None, eStartPos=None, flashback=False, popPk=False): - if flashback and popPk: - raise ValueError('only one of flashback or popPk can be True') +def concat_sql_from_binlogevent(cursor, binlogevent, row=None, eStartPos=None, flashback=False, nopk=False): + if flashback and nopk: + raise ValueError('only one of flashback or nopk can be True') if type(binlogevent) not in (WriteRowsEvent, UpdateRowsEvent, DeleteRowsEvent, QueryEvent): raise ValueError('binlogevent must be WriteRowsEvent, UpdateRowsEvent, DeleteRowsEvent or QueryEvent') @@ -133,7 +133,7 @@ def concat_sql_from_binlogevent(cursor, binlogevent, row=None, eStartPos=None, f sql = cursor.mogrify(template, map(fix_object, row['before_values'].values()+row['after_values'].values())) else: if isinstance(binlogevent, WriteRowsEvent): - if popPk: + if nopk: tableInfo = (binlogevent.table_map)[binlogevent.table_id] if tableInfo.primary_key: row['values'].pop(tableInfo.primary_key) diff --git a/example/FixOldMasterExtraData.md b/example/FixOldMasterExtraData.md index e434ba6..ea44e7f 100644 --- a/example/FixOldMasterExtraData.md +++ b/example/FixOldMasterExtraData.md @@ -41,7 +41,7 @@ MySQL主从切换后如何进行数据修复 我们使用了纯Python开发的工具[binlog2sql](https://github.com/danfengcao/binlog2sql)来操作,执行命令如下 ```bash -$ python binlog2sql.py --popPk -h10.1.1.1 -P3306 -uadmin -p'admin' --start-file='mysql-bin.000040' --start-pos=125466 --end-file='mysql-bin.000041' > oldMaster.sql +$ python binlog2sql.py --no-primary-key -h10.1.1.1 -P3306 -uadmin -p'admin' --start-file='mysql-bin.000040' --start-pos=125466 --end-file='mysql-bin.000041' > oldMaster.sql $ python binlog2sql.py --flashback -h10.1.1.1 -P3306 -uadmin -p'admin' --start-file='mysql-bin.mysql-bin.000040' --start-pos=125466 --end-file='mysql-bin.000041' | mysql -h10.1.1.1 -P3306 -uadmin -p'admin'