Tweak the testcontainers setup

go/base/context.go

@@ -103,6 +103,11 @@ type MigrationContext struct {
 	AzureMySQL               bool
 	AttemptInstantDDL        bool
 
+	// SkipPortValidation allows skipping the port validation in `ValidateConnection`
+	// This is useful when connecting to a MySQL instance where the external port
+	// may not match the internal port.
+	SkipPortValidation bool
+
 	config            ContextConfig
 	configMutex       *sync.Mutex
 	ConfigFile        string

go/base/utils.go

@@ -63,18 +63,27 @@ func StringContainsAll(s string, substrings ...string) bool {
 
 func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, migrationContext *MigrationContext, name string) (string, error) {
 	versionQuery := `select @@global.version`
-	var port, extraPort int
+
 	var version string
 	if err := db.QueryRow(versionQuery).Scan(&version); err != nil {
 		return "", err
 	}
+
+	if migrationContext.SkipPortValidation {
+		return version, nil
+	}
+
+	var extraPort int
+
 	extraPortQuery := `select @@global.extra_port`
 	if err := db.QueryRow(extraPortQuery).Scan(&extraPort); err != nil { //nolint:staticcheck
 		// swallow this error. not all servers support extra_port
 	}
+
 	// AliyunRDS set users port to "NULL", replace it by gh-ost param
 	// GCP set users port to "NULL", replace it by gh-ost param
 	// Azure MySQL set users port to a different value by design, replace it by gh-ost para
+	var port int
 	if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform || migrationContext.AzureMySQL {
 		port = connectionConfig.Key.Port
 	} else {

go/logic/applier_test.go

@@ -19,7 +19,6 @@ import (
 
 	"github.com/github/gh-ost/go/base"
 	"github.com/github/gh-ost/go/binlog"
-	"github.com/github/gh-ost/go/mysql"
 	"github.com/github/gh-ost/go/sql"
 )
 
@@ -183,6 +182,7 @@ func TestApplierBuildDMLEventQuery(t *testing.T) {
 func TestApplierInstantDDL(t *testing.T) {
 	migrationContext := base.NewMigrationContext()
 	migrationContext.DatabaseName = "test"
+	migrationContext.SkipPortValidation = true
 	migrationContext.OriginalTableName = "mytable"
 	migrationContext.AlterStatementOptions = "ADD INDEX (foo)"
 	applier := NewApplier(migrationContext)
@@ -197,14 +197,16 @@ type ApplierTestSuite struct {
 	suite.Suite
 
 	mysqlContainer testcontainers.Container
+	db             *gosql.DB
 }
 
 func (suite *ApplierTestSuite) SetupSuite() {
 	ctx := context.Background()
 	req := testcontainers.ContainerRequest{
-		Image:      "mysql:8.0",
-		Env:        map[string]string{"MYSQL_ROOT_PASSWORD": "root-password"},
-		WaitingFor: wait.ForLog("port: 3306  MySQL Community Server - GPL"),
+		Image:        "mysql:8.0.40",
+		Env:          map[string]string{"MYSQL_ROOT_PASSWORD": "root-password"},
+		ExposedPorts: []string{"3306/tcp"},
+		WaitingFor:   wait.ForListeningPort("3306/tcp"),
 	}
 
 	mysqlContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
@@ -214,47 +216,52 @@ func (suite *ApplierTestSuite) SetupSuite() {
 	suite.Require().NoError(err)
 
 	suite.mysqlContainer = mysqlContainer
+
+	dsn, err := GetDSN(ctx, mysqlContainer)
+	suite.Require().NoError(err)
+
+	db, err := gosql.Open("mysql", dsn)
+	suite.Require().NoError(err)
+
+	suite.db = db
 }
 
 func (suite *ApplierTestSuite) TeardownSuite() {
 	ctx := context.Background()
 
-	suite.Require().NoError(suite.mysqlContainer.Terminate(ctx))
+	suite.Assert().NoError(suite.db.Close())
+	suite.Assert().NoError(suite.mysqlContainer.Terminate(ctx))
 }
 
 func (suite *ApplierTestSuite) SetupTest() {
 	ctx := context.Background()
 
-	rc, _, err := suite.mysqlContainer.Exec(ctx, []string{"mysql", "-uroot", "-proot-password", "-e", "CREATE DATABASE test;"})
-	suite.Require().NoError(err)
-	suite.Require().Equalf(0, rc, "failed to created database: expected exit code 0, got %d", rc)
-
-	rc, _, err = suite.mysqlContainer.Exec(ctx, []string{"mysql", "-uroot", "-proot-password", "-e", "CREATE TABLE test.testing (id INT, item_id INT);"})
+	_, err := suite.db.ExecContext(ctx, "CREATE DATABASE test")
 	suite.Require().NoError(err)
-	suite.Require().Equalf(0, rc, "failed to created table: expected exit code 0, got %d", rc)
 }
 
 func (suite *ApplierTestSuite) TearDownTest() {
 	ctx := context.Background()
 
-	rc, _, err := suite.mysqlContainer.Exec(ctx, []string{"mysql", "-uroot", "-proot-password", "-e", "DROP DATABASE test;"})
+	_, err := suite.db.ExecContext(ctx, "DROP DATABASE test")
 	suite.Require().NoError(err)
-	suite.Require().Equalf(0, rc, "failed to created database: expected exit code 0, got %d", rc)
 }
 
 func (suite *ApplierTestSuite) TestInitDBConnections() {
 	ctx := context.Background()
 
-	host, err := suite.mysqlContainer.ContainerIP(ctx)
+	var err error
+
+	_, err = suite.db.ExecContext(ctx, "CREATE TABLE test.testing (id INT, item_id INT);")
+	suite.Require().NoError(err)
+
+	connectionConfig, err := GetConnectionConfig(ctx, suite.mysqlContainer)
 	suite.Require().NoError(err)
 
 	migrationContext := base.NewMigrationContext()
-	migrationContext.ApplierConnectionConfig = mysql.NewConnectionConfig()
-	migrationContext.ApplierConnectionConfig.Key.Hostname = host
-	migrationContext.ApplierConnectionConfig.Key.Port = 3306
-	migrationContext.ApplierConnectionConfig.User = "root"
-	migrationContext.ApplierConnectionConfig.Password = "root-password"
+	migrationContext.ApplierConnectionConfig = connectionConfig
 	migrationContext.DatabaseName = "test"
+	migrationContext.SkipPortValidation = true
 	migrationContext.OriginalTableName = "testing"
 	migrationContext.SetConnectionConfig("innodb")
 
@@ -274,16 +281,21 @@ func (suite *ApplierTestSuite) TestInitDBConnections() {
 func (suite *ApplierTestSuite) TestApplyDMLEventQueries() {
 	ctx := context.Background()
 
-	host, err := suite.mysqlContainer.ContainerIP(ctx)
+	var err error
+
+	_, err = suite.db.ExecContext(ctx, "CREATE TABLE test.testing (id INT, item_id INT);")
+	suite.Require().NoError(err)
+
+	_, err = suite.db.ExecContext(ctx, "CREATE TABLE test._testing_gho (id INT, item_id INT);")
+	suite.Require().NoError(err)
+
+	connectionConfig, err := GetConnectionConfig(ctx, suite.mysqlContainer)
 	suite.Require().NoError(err)
 
 	migrationContext := base.NewMigrationContext()
-	migrationContext.ApplierConnectionConfig = mysql.NewConnectionConfig()
-	migrationContext.ApplierConnectionConfig.Key.Hostname = host
-	migrationContext.ApplierConnectionConfig.Key.Port = 3306
-	migrationContext.ApplierConnectionConfig.User = "root"
-	migrationContext.ApplierConnectionConfig.Password = "root-password"
+	migrationContext.ApplierConnectionConfig = connectionConfig
 	migrationContext.DatabaseName = "test"
+	migrationContext.SkipPortValidation = true
 	migrationContext.OriginalTableName = "testing"
 	migrationContext.SetConnectionConfig("innodb")
 
@@ -297,10 +309,6 @@ func (suite *ApplierTestSuite) TestApplyDMLEventQueries() {
 	err = applier.InitDBConnections()
 	suite.Require().NoError(err)
 
-	rc, _, err := suite.mysqlContainer.Exec(ctx, []string{"mysql", "-uroot", "-proot-password", "-e", "CREATE TABLE test._testing_gho (id INT, item_id INT);"})
-	suite.Require().NoError(err)
-	suite.Require().Equalf(0, rc, "failed to created table: expected exit code 0, got %d", rc)
-
 	dmlEvents := []*binlog.BinlogDMLEvent{
 		{
 			DatabaseName:    "test",
@@ -313,11 +321,7 @@ func (suite *ApplierTestSuite) TestApplyDMLEventQueries() {
 	suite.Require().NoError(err)
 
 	// Check that the row was inserted
-	db, err := gosql.Open("mysql", "root:root-password@tcp("+host+":3306)/test")
-	suite.Require().NoError(err)
-	defer db.Close()
-
-	rows, err := db.Query("SELECT * FROM test._testing_gho")
+	rows, err := suite.db.Query("SELECT * FROM test._testing_gho")
 	suite.Require().NoError(err)
 	defer rows.Close()
 
@@ -340,16 +344,18 @@ func (suite *ApplierTestSuite) TestApplyDMLEventQueries() {
 func (suite *ApplierTestSuite) TestValidateOrDropExistingTables() {
 	ctx := context.Background()
 
-	host, err := suite.mysqlContainer.ContainerIP(ctx)
+	var err error
+
+	_, err = suite.db.ExecContext(ctx, "CREATE TABLE test.testing (id INT, item_id INT);")
+	suite.Require().NoError(err)
+
+	connectionConfig, err := GetConnectionConfig(ctx, suite.mysqlContainer)
 	suite.Require().NoError(err)
 
 	migrationContext := base.NewMigrationContext()
-	migrationContext.ApplierConnectionConfig = mysql.NewConnectionConfig()
-	migrationContext.ApplierConnectionConfig.Key.Hostname = host
-	migrationContext.ApplierConnectionConfig.Key.Port = 3306
-	migrationContext.ApplierConnectionConfig.User = "root"
-	migrationContext.ApplierConnectionConfig.Password = "root-password"
+	migrationContext.ApplierConnectionConfig = connectionConfig
 	migrationContext.DatabaseName = "test"
+	migrationContext.SkipPortValidation = true
 	migrationContext.OriginalTableName = "testing"
 	migrationContext.SetConnectionConfig("innodb")
 
@@ -367,6 +373,130 @@ func (suite *ApplierTestSuite) TestValidateOrDropExistingTables() {
 	suite.Require().NoError(err)
 }
 
+func (suite *ApplierTestSuite) TestValidateOrDropExistingTablesWithGhostTableExisting() {
+	ctx := context.Background()
+
+	var err error
+
+	_, err = suite.db.ExecContext(ctx, "CREATE TABLE test.testing (id INT, item_id INT);")
+	suite.Require().NoError(err)
+
+	_, err = suite.db.ExecContext(ctx, "CREATE TABLE test._testing_gho (id INT, item_id INT);")
+	suite.Require().NoError(err)
+
+	connectionConfig, err := GetConnectionConfig(ctx, suite.mysqlContainer)
+	suite.Require().NoError(err)
+
+	migrationContext := base.NewMigrationContext()
+	migrationContext.ApplierConnectionConfig = connectionConfig
+	migrationContext.DatabaseName = "test"
+	migrationContext.SkipPortValidation = true
+	migrationContext.OriginalTableName = "testing"
+	migrationContext.SetConnectionConfig("innodb")
+
+	migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "item_id"})
+	migrationContext.SharedColumns = sql.NewColumnList([]string{"id", "item_id"})
+	migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id", "item_id"})
+
+	applier := NewApplier(migrationContext)
+	defer applier.Teardown()
+
+	err = applier.InitDBConnections()
+	suite.Require().NoError(err)
+
+	err = applier.ValidateOrDropExistingTables()
+	suite.Require().Error(err)
+	suite.Require().EqualError(err, "Table `_testing_gho` already exists. Panicking. Use --initially-drop-ghost-table to force dropping it, though I really prefer that you drop it or rename it away")
+}
+
+func (suite *ApplierTestSuite) TestValidateOrDropExistingTablesWithGhostTableExistingAndInitiallyDropGhostTableSet() {
+	ctx := context.Background()
+
+	var err error
+
+	_, err = suite.db.ExecContext(ctx, "CREATE TABLE test.testing (id INT, item_id INT);")
+	suite.Require().NoError(err)
+
+	_, err = suite.db.ExecContext(ctx, "CREATE TABLE test._testing_gho (id INT, item_id INT);")
+	suite.Require().NoError(err)
+
+	connectionConfig, err := GetConnectionConfig(ctx, suite.mysqlContainer)
+	suite.Require().NoError(err)
+
+	migrationContext := base.NewMigrationContext()
+	migrationContext.ApplierConnectionConfig = connectionConfig
+	migrationContext.DatabaseName = "test"
+	migrationContext.SkipPortValidation = true
+	migrationContext.OriginalTableName = "testing"
+	migrationContext.SetConnectionConfig("innodb")
+
+	migrationContext.InitiallyDropGhostTable = true
+
+	applier := NewApplier(migrationContext)
+	defer applier.Teardown()
+
+	err = applier.InitDBConnections()
+	suite.Require().NoError(err)
+
+	err = applier.ValidateOrDropExistingTables()
+	suite.Require().NoError(err)
+
+	// Check that the ghost table was dropped
+	var tableName string
+	//nolint:execinquery
+	err = suite.db.QueryRow("SHOW TABLES IN test LIKE '_testing_gho'").Scan(&tableName)
+	suite.Require().Error(err)
+	suite.Require().Equal(gosql.ErrNoRows, err)
+}
+
+func (suite *ApplierTestSuite) TestCreateGhostTable() {
+	ctx := context.Background()
+
+	var err error
+
+	_, err = suite.db.ExecContext(ctx, "CREATE TABLE test.testing (id INT, item_id INT);")
+	suite.Require().NoError(err)
+
+	connectionConfig, err := GetConnectionConfig(ctx, suite.mysqlContainer)
+	suite.Require().NoError(err)
+
+	migrationContext := base.NewMigrationContext()
+	migrationContext.ApplierConnectionConfig = connectionConfig
+	migrationContext.DatabaseName = "test"
+	migrationContext.SkipPortValidation = true
+	migrationContext.OriginalTableName = "testing"
+	migrationContext.SetConnectionConfig("innodb")
+
+	migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "item_id"})
+	migrationContext.SharedColumns = sql.NewColumnList([]string{"id", "item_id"})
+	migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id", "item_id"})
+
+	migrationContext.InitiallyDropGhostTable = true
+
+	applier := NewApplier(migrationContext)
+	defer applier.Teardown()
+
+	err = applier.InitDBConnections()
+	suite.Require().NoError(err)
+
+	err = applier.CreateGhostTable()
+	suite.Require().NoError(err)
+
+	// Check that the ghost table was created
+	var tableName string
+	//nolint:execinquery
+	err = suite.db.QueryRow("SHOW TABLES IN test LIKE '_testing_gho'").Scan(&tableName)
+	suite.Require().NoError(err)
+	suite.Require().Equal("_testing_gho", tableName)
+
+	// Check that the ghost table has the same columns as the original table
+	var createDDL string
+	//nolint:execinquery
+	err = suite.db.QueryRow("SHOW CREATE TABLE test._testing_gho").Scan(&tableName, &createDDL)
+	suite.Require().NoError(err)
+	suite.Require().Equal("CREATE TABLE `_testing_gho` (\n  `id` int DEFAULT NULL,\n  `item_id` int DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci", createDDL)
+}
+
 func TestApplier(t *testing.T) {
 	suite.Run(t, new(ApplierTestSuite))
 }