Fix no table is replicated when excludeTableRegex is set while includeTableRegex is nil (#874)

* Fix bug when excludeTableRegex is set while includeTableRegex is nil

* add TestIncludeExcludeTableRegex

* fix typo

* Fix TestIncludeExcludeTableRegex

* improve IncludeTableRegex and ExcludeTableRegex comment

---------

Co-authored-by: Jijun Gao <jjgao@clustertech.com>
Co-authored-by: lance6716 <lance6716@gmail.com>

Author: 3 people

canal/canal.go

@@ -99,13 +99,20 @@ func NewCanal(cfg *Config) (*Canal, error) {
 		return nil, errors.Trace(err)
 	}
 
-	// init table filter
+	if err := c.initTableFilter(); err != nil {
+		return nil, errors.Trace(err)
+	}
+
+	return c, nil
+}
+
+func (c *Canal) initTableFilter() error {
 	if n := len(c.cfg.IncludeTableRegex); n > 0 {
 		c.includeTableRegex = make([]*regexp.Regexp, n)
 		for i, val := range c.cfg.IncludeTableRegex {
 			reg, err := regexp.Compile(val)
 			if err != nil {
-				return nil, errors.Trace(err)
+				return errors.Trace(err)
 			}
 			c.includeTableRegex[i] = reg
 		}
@@ -116,7 +123,7 @@ func NewCanal(cfg *Config) (*Canal, error) {
 		for i, val := range c.cfg.ExcludeTableRegex {
 			reg, err := regexp.Compile(val)
 			if err != nil {
-				return nil, errors.Trace(err)
+				return errors.Trace(err)
 			}
 			c.excludeTableRegex[i] = reg
 		}
@@ -125,8 +132,7 @@ func NewCanal(cfg *Config) (*Canal, error) {
 	if c.includeTableRegex != nil || c.excludeTableRegex != nil {
 		c.tableMatchCache = make(map[string]bool)
 	}
-
-	return c, nil
+	return nil
 }
 
 func (c *Canal) prepareDumper() error {
@@ -143,7 +149,7 @@ func (c *Canal) prepareDumper() error {
 	}
 
 	if c.dumper == nil {
-		//no mysqldump, use binlog only
+		// no mysqldump, use binlog only
 		return nil
 	}
 
@@ -293,7 +299,10 @@ func (c *Canal) checkTableMatch(key string) bool {
 				break
 			}
 		}
+	} else {
+		matchFlag = true
 	}
+
 	// check exclude
 	if matchFlag && c.excludeTableRegex != nil {
 		for _, reg := range c.excludeTableRegex {

canal/canal_test.go

@@ -405,3 +405,29 @@ func TestDropIndexExp(t *testing.T) {
 		}
 	}
 }
+
+func TestIncludeExcludeTableRegex(t *testing.T) {
+	cfg := NewDefaultConfig()
+
+	// include & exclude config
+	cfg.IncludeTableRegex = make([]string, 1)
+	cfg.IncludeTableRegex[0] = ".*\\.canal_test"
+	cfg.ExcludeTableRegex = make([]string, 2)
+	cfg.ExcludeTableRegex[0] = "mysql\\..*"
+	cfg.ExcludeTableRegex[1] = ".*\\..*_inner"
+
+	c := new(Canal)
+	c.cfg = cfg
+	require.Nil(t, c.initTableFilter())
+	require.True(t, c.checkTableMatch("test.canal_test"))
+	require.False(t, c.checkTableMatch("test.canal_test_inner"))
+	require.False(t, c.checkTableMatch("mysql.canal_test_inner"))
+
+	cfg.IncludeTableRegex = nil
+	c = new(Canal)
+	c.cfg = cfg
+	require.Nil(t, c.initTableFilter())
+	require.True(t, c.checkTableMatch("test.canal_test"))
+	require.False(t, c.checkTableMatch("test.canal_test_inner"))
+	require.False(t, c.checkTableMatch("mysql.canal_test_inner"))
+}

canal/config.go

@@ -61,10 +61,12 @@ type Config struct {
 	HeartbeatPeriod time.Duration `toml:"heartbeat_period"`
 	ReadTimeout     time.Duration `toml:"read_timeout"`
 
-	// IncludeTableRegex or ExcludeTableRegex should contain database name
+	// IncludeTableRegex or ExcludeTableRegex should contain database name.
+	// IncludeTableRegex defines the tables that will be included, if empty, all tables will be included.
+	// ExcludeTableRegex defines the tables that will be excluded from the ones defined by IncludeTableRegex.
 	// Only a table which matches IncludeTableRegex and dismatches ExcludeTableRegex will be processed
 	// eg, IncludeTableRegex : [".*\\.canal"], ExcludeTableRegex : ["mysql\\..*"]
-	//     this will include all database's 'canal' table, except database 'mysql'
+	//     this will include all database's 'canal' table, except database 'mysql'.
 	// Default IncludeTableRegex and ExcludeTableRegex are empty, this will include all tables
 	IncludeTableRegex []string `toml:"include_table_regex"`
 	ExcludeTableRegex []string `toml:"exclude_table_regex"`