refactor(move-tables): clean up prototype phase 1.2 skip-tables (#8206)

go/base/context.go

@@ -271,6 +271,17 @@ type MigrationContext struct {
 	SkipMetadataLockCheck             bool
 	IsOpenMetadataLockInstruments     bool
 
+	// move tables:
+	MoveTables struct {
+		TableNames       []string // List of table names to be moved.
+		TargetHost       string   // Target hostname for the move. This must be a primary/writable host.
+		TargetPort       int      // Target MySQL port for the move.
+		TargetUser       string   // Target username for the move. If not specified, it will default to the source user.
+		TargetPass       string   // Target password for the move. If not specified, it will default to the source password.
+		TargetDatabase   string   // Target database name for the move. If not specified, it will default to the source database name.
+		ConnectionConfig *mysql.ConnectionConfig
+	}
+
 	Log Logger
 }
 
@@ -343,6 +354,9 @@ func (mctx *MigrationContext) SetConnectionConfig(storageEngine string) error {
 	}
 	mctx.InspectorConnectionConfig.TransactionIsolation = transactionIsolation
 	mctx.ApplierConnectionConfig.TransactionIsolation = transactionIsolation
+	if mctx.MoveTables.ConnectionConfig != nil {
+		mctx.MoveTables.ConnectionConfig.TransactionIsolation = transactionIsolation
+	}
 	return nil
 }
 
@@ -353,6 +367,9 @@ func (mctx *MigrationContext) SetConnectionCharset(charset string) {
 
 	mctx.InspectorConnectionConfig.Charset = charset
 	mctx.ApplierConnectionConfig.Charset = charset
+	if mctx.MoveTables.ConnectionConfig != nil {
+		mctx.MoveTables.ConnectionConfig.Charset = charset
+	}
 }
 
 func getSafeTableName(baseName string, suffix string) string {
@@ -378,6 +395,24 @@ func (mctx *MigrationContext) GetGhostTableName() string {
 	}
 }
 
+// GetTargetTableName generates the name of the target table, based on original table name and
+// the migration context (i.e. move-tables mode).
+func (mctx *MigrationContext) GetTargetTableName() string {
+	if mctx.IsMoveTablesMode() {
+		return mctx.MoveTables.TableNames[0]
+	}
+	return mctx.GetGhostTableName()
+}
+
+// GetTargetDatabaseName fetches the name of the target database, which defaults to the original
+// database name unless we're in move-tables mode.
+func (mctx *MigrationContext) GetTargetDatabaseName() string {
+	if mctx.IsMoveTablesMode() {
+		return mctx.DatabaseName
+	}
+	return mctx.MoveTables.TargetDatabase
+}
+
 // GetOldTableName generates the name of the "old" table, into which the original table is renamed.
 func (mctx *MigrationContext) GetOldTableName() string {
 	var tableName string
@@ -900,8 +935,15 @@ func (mctx *MigrationContext) AddThrottleControlReplicaKey(key mysql.InstanceKey
 	return nil
 }
 
+// TODO(chriskirkland): pick up from here:
+//
+// need to figure out how to populate the applier connection config from
+// the inspector connection config but using the "target" values from the CLI
+// args.
+
 // ApplyCredentials sorts out the credentials between the config file and the CLI flags
 func (mctx *MigrationContext) ApplyCredentials() {
+	//TODO(chriskirkland): make this work for move-tables
 	mctx.configMutex.Lock()
 	defer mctx.configMutex.Unlock()
 
@@ -919,9 +961,23 @@ func (mctx *MigrationContext) ApplyCredentials() {
 		// Override
 		mctx.InspectorConnectionConfig.Password = mctx.CliPassword
 	}
+
+	if mctx.IsMoveTablesMode() {
+		// apply credentials for the applier from target CLI args
+		if mctx.MoveTables.ConnectionConfig == nil {
+			mctx.MoveTables.ConnectionConfig = &mysql.ConnectionConfig{}
+		}
+		mctx.MoveTables.ConnectionConfig.User = mctx.MoveTables.TargetUser
+		mctx.MoveTables.ConnectionConfig.Password = mctx.MoveTables.TargetPass
+		mctx.MoveTables.ConnectionConfig.Key = mysql.InstanceKey{
+			Hostname: mctx.MoveTables.TargetHost,
+			Port:     mctx.MoveTables.TargetPort,
+		}
+	}
 }
 
 func (mctx *MigrationContext) SetupTLS() error {
+	//TODO(chriskirkland): make this work for move-tables?
 	if mctx.UseTLS {
 		return mctx.InspectorConnectionConfig.UseTLS(mctx.TLSCACertificate, mctx.TLSCertificate, mctx.TLSKey, mctx.TLSAllowInsecure)
 	}
@@ -1029,6 +1085,11 @@ func (mctx *MigrationContext) CancelContext() {
 	}
 }
 
+// IsMoveTablesMode returns true if gh-ost should be used for moving tables instead of running a schema migration.
+func (mctx *MigrationContext) IsMoveTablesMode() bool {
+	return len(mctx.MoveTables.TableNames) > 0
+}
+
 // SendWithContext attempts to send a value to a channel, but returns early
 // if the context is cancelled. This prevents goroutine deadlocks when the
 // channel receiver has exited due to an error.

go/cmd/gh-ost/main.go

@@ -12,10 +12,12 @@ import (
 	"os"
 	"os/signal"
 	"regexp"
+	"strings"
 	"syscall"
 
 	"github.com/github/gh-ost/go/base"
 	"github.com/github/gh-ost/go/logic"
+	"github.com/github/gh-ost/go/mysql"
 	"github.com/github/gh-ost/go/sql"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/openark/golib/log"
@@ -164,8 +166,16 @@ func main() {
 	version := flag.Bool("version", false, "Print version & exit")
 	checkFlag := flag.Bool("check-flag", false, "Check if another flag exists/supported. This allows for cross-version scripting. Exits with 0 when all additional provided flags exist, nonzero otherwise. You must provide (dummy) values for flags that require a value. Example: gh-ost --check-flag --cut-over-lock-timeout-seconds --nice-ratio 0")
 	flag.StringVar(&migrationContext.ForceTmpTableName, "force-table-names", "", "table name prefix to be used on the temporary tables")
-	flag.CommandLine.SetOutput(os.Stdout)
 
+	// move tables flags
+	moveTables := flag.String("move-tables", "", "Comma delimited list of tables to move. e.g. 'table1,table2,table3'. This is a special mode that allows you to move tables between database clusters. This mode is mutually exclusive with --alter, --table, --test-on-replica, --migrate-on-replica, --execute-on-replica, and --revert.")
+	flag.StringVar(&migrationContext.MoveTables.TargetHost, "target-host", "", "Target MySQL hostname for --move-tables mode. Must be specified if --move-tables is specified.")
+	flag.IntVar(&migrationContext.MoveTables.TargetPort, "target-port", 3306, "Target MySQL port for --move-tables mode. Defaults to 3306.")
+	flag.StringVar(&migrationContext.MoveTables.TargetUser, "target-user", "", "Target MySQL username for --move-tables mode. If not provided, uses the same user as the source connection")
+	flag.StringVar(&migrationContext.MoveTables.TargetPass, "target-password", "", "Target MySQL password for --move-tables mode. If not provided, uses the same password as the source connection")
+	flag.StringVar(&migrationContext.MoveTables.TargetDatabase, "target-database", "", "Target MySQL database name for --move-tables mode. If not provided, uses the same database name as the source connection")
+
+	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Parse()
 
 	if *checkFlag {
@@ -203,13 +213,7 @@ func main() {
 		migrationContext.Log.SetLevel(log.ERROR)
 	}
 
-	if err := migrationContext.SetConnectionConfig(*storageEngine); err != nil {
-		migrationContext.Log.Fatale(err)
-	}
-
-	migrationContext.SetConnectionCharset(*charset)
-
-	if migrationContext.AlterStatement == "" && !migrationContext.Revert {
+	if migrationContext.AlterStatement == "" && !migrationContext.Revert && *moveTables == "" {
 		log.Fatal("--alter must be provided and statement must not be empty")
 	}
 	parser := sql.NewParserFromAlterStatement(migrationContext.AlterStatement)
@@ -250,7 +254,7 @@ func main() {
 		migrationContext.Log.Fatale(err)
 	}
 
-	if migrationContext.OriginalTableName == "" {
+	if migrationContext.OriginalTableName == "" && *moveTables == "" {
 		if parser.HasExplicitTable() {
 			migrationContext.OriginalTableName = parser.GetExplicitTable()
 		} else {
@@ -320,6 +324,40 @@ func main() {
 		migrationContext.Log.Warning("--exact-rowcount with --panic-on-warnings: row counts cannot be exact due to warning detection")
 	}
 
+	if *moveTables != "" {
+		if migrationContext.AlterStatement != "" {
+			log.Fatal("--move-tables is mutually exclusive with --alter")
+		}
+		if migrationContext.OriginalTableName != "" {
+			log.Fatal("--move-tables is mutually exclusive with --table")
+		}
+		if migrationContext.TestOnReplica {
+			log.Fatal("--move-tables is mutually exclusive with --test-on-replica")
+		}
+		if migrationContext.MigrateOnReplica {
+			log.Fatal("--move-tables is mutually exclusive with --migrate-on-replica")
+		}
+		if migrationContext.Revert {
+			log.Fatal("--move-tables is mutually exclusive with --revert")
+		}
+		if migrationContext.MoveTables.TargetHost == "" {
+			log.Fatal("--target-host must be specified when using --move-tables")
+		}
+		migrationContext.MoveTables.TableNames = strings.Split(*moveTables, ",")
+		if len(migrationContext.MoveTables.TableNames) > 1 {
+			// Future version will support moving multiple tables at the same time.
+			// For now, we only support moving a single table at a time.
+			log.Fatal("--move-tables currently supports only a single table")
+		}
+		migrationContext.MoveTables.ConnectionConfig = mysql.NewConnectionConfig()
+	}
+
+	if err := migrationContext.SetConnectionConfig(*storageEngine); err != nil {
+		migrationContext.Log.Fatale(err)
+	}
+
+	migrationContext.SetConnectionCharset(*charset)
+
 	switch *cutOver {
 	case "atomic", "default", "":
 		migrationContext.CutOverType = base.CutOverAtomic
@@ -379,6 +417,8 @@ func main() {
 	var err error
 	if migrationContext.Revert {
 		err = migrator.Revert()
+	} else if migrationContext.IsMoveTablesMode() {
+		err = migrator.MoveTables()
 	} else {
 		err = migrator.Migrate()
 	}