loader, syncer: Update sql_mode to more compatible (#1869)

loader/loader.go

@@ -528,10 +528,17 @@ func (l *Loader) Init(ctx context.Context) (err error) {
 			break
 		}
 	}
+
 	if !hasSQLMode {
-		lcfg.To.Session["sql_mode"] = l.cfg.LoaderConfig.SQLMode
+		sqlModes, err3 := utils.AdjustSQLModeCompatible(l.cfg.LoaderConfig.SQLMode)
+		if err3 != nil {
+			l.logger.Warn("cannot adjust sql_mode compatible, the sql_mode will stay the same", log.ShortError(err3))
+		}
+		lcfg.To.Session["sql_mode"] = sqlModes
 	}
 
+	l.logger.Info("loader's sql_mode is", zap.String("sqlmode", lcfg.To.Session["sql_mode"]))
+
 	l.toDB, l.toDBConns, err = createConns(tctx, lcfg, l.cfg.PoolSize)
 	if err != nil {
 		return err

pkg/utils/db.go

@@ -524,3 +524,56 @@ func AddGSetWithPurged(ctx context.Context, gset gtid.Set, conn *sql.Conn) (gtid
 	_ = ret.Set(newGset)
 	return ret, nil
 }
+
+// AdjustSQLModeCompatible adjust downstream sql mode to compatible.
+// TODO: When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00
+// and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31,
+// as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'.
+// This is because the implementation of go-mysql, that you can see
+// https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087
+func AdjustSQLModeCompatible(sqlModes string) (string, error) {
+	needDisable := []string{
+		"NO_ZERO_IN_DATE",
+		"NO_ZERO_DATE",
+		"ERROR_FOR_DIVISION_BY_ZERO",
+		"NO_AUTO_CREATE_USER",
+		"STRICT_TRANS_TABLES",
+		"STRICT_ALL_TABLES",
+	}
+	needEnable := []string{
+		"IGNORE_SPACE",
+		"NO_AUTO_VALUE_ON_ZERO",
+		"ALLOW_INVALID_DATES",
+	}
+	disable := strings.Join(needDisable, ",")
+	enable := strings.Join(needEnable, ",")
+
+	mode, err := tmysql.GetSQLMode(sqlModes)
+	if err != nil {
+		return sqlModes, err
+	}
+	disableMode, err2 := tmysql.GetSQLMode(disable)
+	if err2 != nil {
+		return sqlModes, err2
+	}
+	enableMode, err3 := tmysql.GetSQLMode(enable)
+	if err3 != nil {
+		return sqlModes, err3
+	}
+	// About this bit manipulation, details can be seen
+	// https://github.com/pingcap/dm/pull/1869#discussion_r669771966
+	mode = (mode &^ disableMode) | enableMode
+
+	return GetSQLModeStrBySQLMode(mode), nil
+}
+
+// GetSQLModeStrBySQLMode get string represent of sql_mode by sql_mode.
+func GetSQLModeStrBySQLMode(sqlMode tmysql.SQLMode) string {
+	var sqlModeStr []string
+	for str, SQLMode := range tmysql.Str2SQLMode {
+		if sqlMode&SQLMode != 0 {
+			sqlModeStr = append(sqlModeStr, str)
+		}
+	}
+	return strings.Join(sqlModeStr, ",")
+}

syncer/syncer.go

@@ -2860,10 +2860,13 @@ func (s *Syncer) createDBs(ctx context.Context) error {
 	if !hasSQLMode {
 		sqlMode, err2 := utils.GetGlobalVariable(ctx, s.fromDB.BaseDB.DB, "sql_mode")
 		if err2 != nil {
-			s.tctx.L().Warn("cannot get sql_mode from upstream database", log.ShortError(err2))
-		} else {
-			s.cfg.To.Session["sql_mode"] = sqlMode
+			s.tctx.L().Warn("cannot get sql_mode from upstream database, the sql_mode will be assigned \"IGNORE_SPACE, NO_AUTO_VALUE_ON_ZERO, ALLOW_INVALID_DATES\"", log.ShortError(err2))
+		}
+		sqlModes, err3 := utils.AdjustSQLModeCompatible(sqlMode)
+		if err3 != nil {
+			s.tctx.L().Warn("cannot adjust sql_mode compatible, the sql_mode will be assigned  stay the same", log.ShortError(err3))
 		}
+		s.cfg.To.Session["sql_mode"] = sqlModes
 	}
 
 	dbCfg = s.cfg.To

tests/handle_error/conf/double-source-no-sharding.yaml

@@ -10,6 +10,9 @@ target-database:
   user: "test"
   password: "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs="
 
+  session:
+    sql_mode: "STRICT_TRANS_TABLES"
+
 mysql-instances:
   - source-id: "mysql-replica-01"
     block-allow-list:  "instance"

tests/others_integration.txt

@@ -12,3 +12,4 @@ expression_filter
 fake_rotate_event
 metrics
 case_sensitive
+sql_mode

tests/sql_mode/conf/diff_config.toml

@@ -0,0 +1,56 @@
+# diff Configuration.
+
+log-level = "info"
+
+chunk-size = 20
+
+check-thread-count = 4
+
+sample-percent = 100
+
+use-checksum = true
+
+fix-sql-file = "fix.sql"
+
+# tables need to check.
+[[check-tables]]
+schema = "sql_mode"
+tables = ["~.*"]
+
+[[source-db]]
+host = "127.0.0.1"
+port = 3306
+user = "root"
+password = "123456"
+instance-id = "source-1"
+
+[[table-config]]
+schema = "sql_mode"
+table = "t_1"
+
+[[table-config.source-tables]]
+instance-id = "source-1"
+schema = "sql_mode"
+table  = "t_1"
+
+[[source-db]]
+host = "127.0.0.1"
+port = 3307
+user = "root"
+password = "123456"
+instance-id = "source-2"
+
+[[table-config]]
+schema = "sql_mode"
+table = "t_2"
+
+[[table-config.source-tables]]
+instance-id = "source-2"
+schema = "sql_mode"
+table  = "t_2"
+
+[target-db]
+host = "127.0.0.1"
+port = 4000
+user = "test"
+password = "123456"

tests/sql_mode/conf/dm-master.toml

@@ -0,0 +1,4 @@
+# Master Configuration.
+master-addr = ":8261"
+advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/sql_mode/conf/dm-task.yaml

@@ -0,0 +1,46 @@
+---
+name: sql_mode
+task-mode: all
+is-sharding: false
+meta-schema: "dm_meta"
+enable-heartbeat: false
+
+target-database:
+  host: "127.0.0.1"
+  port: 4000
+  user: "root"
+  password: ""
+
+mysql-instances:
+  - source-id: "mysql-replica-01"
+    block-allow-list:  "instance"
+    mydumper-config-name: "global"
+    loader-config-name: "global"
+    syncer-config-name: "global"
+
+  - source-id: "mysql-replica-02"
+    block-allow-list:  "instance"
+    mydumper-config-name: "global"
+    loader-config-name: "global"
+    syncer-config-name: "global"
+
+block-allow-list:
+  instance:
+    do-dbs: ["sql_mode"]
+
+mydumpers:
+  global:
+    threads: 4
+    chunk-filesize: 0
+    skip-tz-utc: true
+    extra-args: "--statement-size=4000"
+
+loaders:
+  global:
+    pool-size: 16
+    dir: "./dumped_data"
+
+syncers:
+  global:
+    worker-count: 16
+    batch: 100

tests/sql_mode/conf/dm-worker1.toml

@@ -0,0 +1,2 @@
+name = "worker1"
+join = "127.0.0.1:8261"

tests/sql_mode/conf/dm-worker2.toml

@@ -0,0 +1,2 @@
+name = "worker2"
+join = "127.0.0.1:8261"

tests/sql_mode/conf/source1.yaml

@@ -0,0 +1,11 @@
+source-id: mysql-replica-01
+flavor: ''
+enable-gtid: false
+enable-relay: true
+relay-binlog-name: ''
+relay-binlog-gtid: ''
+from:
+  host: 127.0.0.1
+  user: root
+  password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=
+  port: 3306

tests/sql_mode/conf/source2.yaml

@@ -0,0 +1,11 @@
+source-id: mysql-replica-02
+flavor: ''
+enable-gtid: false
+enable-relay: true
+relay-binlog-name: ''
+relay-binlog-gtid: ''
+from:
+  host: 127.0.0.1
+  user: root
+  password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=
+  port: 3307

tests/sql_mode/data/db1.increment.sql

@@ -0,0 +1,32 @@
+set @@session.SQL_MODE='';
+use `sql_mode`;
+
+-- test sql_mode PIPES_AS_CONCAT
+insert into t_1(num) values('pipes'||'as'||'concat');
+
+-- test sql_mode ANSI_QUOTES
+insert into t_1(name) values("a");
+
+-- test sql_mode IGNORE_SPACE
+create table sum (id int not null, primary key(id));
+
+-- test sql_mode NO_AUTO_VALUE_ON_ZERO
+insert into t_1(id, name) values (30, 'a');
+insert into t_1(id, name) values (0, 'b');
+insert into t_1(id, name) values (0, 'c');
+
+-- test sql_mode NO_BACKSLASH_ESCAPES
+insert into t_1(name) values ('\\b');
+
+-- test sql_mode STRICT_TRANS_TABLES && STRICT_ALL_TABLES && NO_ZERO_IN_DATE && NO_ZERO_DATE && ALLOW_INVALID_DATES
+insert into t_1(dt) values('0000-06-00');
+insert into t_1(dt) values('0000-00-01');
+insert into t_1(dt) values('0000-06-01');
+insert into t_1(dt) values('0000-00-00');
+
+-- test sql_mode ERROR_FOR_DIVISION_BY_ZERO
+insert into t_1(num) values(4/0);
+
+-- test sql_mode NO_AUTO_CREATE_USER
+drop user if exists 'no_auto_create_user';
+grant select on *.* to 'no_auto_create_user';

tests/sql_mode/data/db1.prepare.sql

@@ -0,0 +1,42 @@
+set @@session.SQL_MODE='';
+
+drop database if exists `sql_mode`;
+create database `sql_mode`;
+use `sql_mode`;
+CREATE TABLE `t_1` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  `name` varchar(60),
+  `num` int,
+  `dt` datetime,
+  PRIMARY KEY (id)
+);
+
+-- test sql_mode PIPES_AS_CONCAT
+insert into t_1(num) values('pipes'||'as'||'concat');
+
+-- test sql_mode ANSI_QUOTES
+insert into t_1(name) values("a");
+
+-- test sql_mode IGNORE_SPACE
+create table count (id int not null, primary key(id));
+
+-- test sql_mode NO_AUTO_VALUE_ON_ZERO
+insert into t_1(id, name) values (10, 'a');
+insert into t_1(id, name) values (0, 'b');
+insert into t_1(id, name) values (0, 'c');
+
+-- test sql_mode NO_BACKSLASH_ESCAPES
+insert into t_1(name) values ('\\a');
+
+-- test sql_mode STRICT_TRANS_TABLES && STRICT_ALL_TABLES && NO_ZERO_IN_DATE && NO_ZERO_DATE && ALLOW_INVALID_DATES
+insert into t_1(dt) values('0000-06-00');
+insert into t_1(dt) values('0000-00-01');
+insert into t_1(dt) values('0000-06-01');
+insert into t_1(dt) values('0000-00-00');
+
+-- test sql_mode ERROR_FOR_DIVISION_BY_ZERO
+insert into t_1(num) values(4/0);
+
+-- test sql_mode NO_AUTO_CREATE_USER
+drop user if exists 'no_auto_create_user';
+grant select on *.* to 'no_auto_create_user';

tests/sql_mode/data/db2.increment.sql

@@ -0,0 +1,24 @@
+set @@session.sql_mode='ONLY_FULL_GROUP_BY,NO_UNSIGNED_SUBTRACTION,NO_DIR_IN_CREATE,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ALLOW_INVALID_DATES,ERROR_FOR_DIVISION_BY_ZERO,HIGH_NOT_PRECEDENCE,NO_ENGINE_SUBSTITUTION,REAL_AS_FLOAT';
+-- NO_AUTO_CREATE_USER set failed in mysql8.0
+
+use sql_mode;
+
+-- test sql_mode PIPES_AS_CONCAT
+set @@session.sql_mode=concat(@@session.sql_mode, ',PIPES_AS_CONCAT');
+insert into t_2(name) values('pipes'||'as'||'concat');
+
+-- test sql_mode ANSI_QUOTES
+insert into t_2(name) values("a");
+
+-- test sql_mode IGNORE_SPACE
+set @@session.sql_mode=concat(@@session.sql_mode, ',IGNORE_SPACE');
+insert into t_2(name) values(concat ('ignore', 'space'));
+
+-- test sql_mode NO_AUTO_VALUE_ON_ZERO
+set @@session.sql_mode=concat(@@session.sql_mode, ',NO_AUTO_VALUE_ON_ZERO');
+insert into t_2(id, name) values (20, 'a');
+replace into t_2(id, name) values (0, 'c');
+
+-- test sql_mode NO_BACKSLASH_ESCAPES
+set @@session.sql_mode=concat(@@session.sql_mode, ',NO_BACKSLASH_ESCAPES');
+insert into t_2(name) values ('\\a');

tests/sql_mode/data/db2.prepare.sql

@@ -0,0 +1,33 @@
+set @@session.sql_mode='ONLY_FULL_GROUP_BY,NO_UNSIGNED_SUBTRACTION,NO_DIR_IN_CREATE,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ALLOW_INVALID_DATES,ERROR_FOR_DIVISION_BY_ZERO,HIGH_NOT_PRECEDENCE,NO_ENGINE_SUBSTITUTION,REAL_AS_FLOAT';
+-- NO_AUTO_CREATE_USER set failed in mysql8.0
+
+drop database if exists `sql_mode`;
+create database `sql_mode`;
+use `sql_mode`;
+CREATE TABLE `t_2` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  `name` varchar(60),
+  `num` int,
+  `dt` datetime,
+  PRIMARY KEY (id)
+);
+
+-- test sql_mode PIPES_AS_CONCAT
+set @@session.sql_mode=concat(@@session.sql_mode, ',PIPES_AS_CONCAT');
+insert into t_2(name) values('pipes'||'as'||'concat');
+
+-- test sql_mode ANSI_QUOTES
+insert into t_2(name) values("a");
+
+-- test sql_mode IGNORE_SPACE
+set @@session.sql_mode=concat(@@session.sql_mode, ',IGNORE_SPACE');
+insert into t_2(name) values(concat ('ignore', 'space'));
+
+-- test sql_mode NO_AUTO_VALUE_ON_ZERO
+set @@session.sql_mode=concat(@@session.sql_mode, ',NO_AUTO_VALUE_ON_ZERO');
+insert into t_2(id, name) values (10, 'a');
+insert into t_2(id, name) values (0, 'b');
+
+-- test sql_mode NO_BACKSLASH_ESCAPES
+set @@session.sql_mode=concat(@@session.sql_mode, ',NO_BACKSLASH_ESCAPES');
+insert into t_2(name) values ('\\a');