Optionally allow use of stored procedures for MySql to drop columns in db migration

2026-10-23 10:03:59 +00:00 · 2021-09-27 17:08:22 +02:00
parent 033c579984
commit 091b5fbf26
17 changed files with 474 additions and 7 deletions
@@ -66,6 +66,8 @@ public class PlatformConfig {

  private boolean caseSensitiveCollation = true;

+  private boolean useMigrationStoredProcedures = false;
+
  /**
   * Modify the default mapping of standard types such as default precision for DECIMAL etc.
   */
@@ -90,6 +92,7 @@ public class PlatformConfig {
    this.geometrySRID = platformConfig.geometrySRID;
    this.dbUuid = platformConfig.dbUuid;
    this.caseSensitiveCollation = platformConfig.caseSensitiveCollation;
+    this.useMigrationStoredProcedures = platformConfig.useMigrationStoredProcedures;
    this.allQuotedIdentifiers = platformConfig.allQuotedIdentifiers;
    this.databaseInetAddressVarchar = platformConfig.databaseInetAddressVarchar;
    this.customDbTypeMappings = platformConfig.customDbTypeMappings;
@@ -139,6 +142,20 @@ public class PlatformConfig {
    this.caseSensitiveCollation = caseSensitiveCollation;
  }

+  /**
+   * Return true if force use of helper stored procedures for migrations.
+   */
+  public boolean isUseMigrationStoredProcedures() {
+    return useMigrationStoredProcedures;
+  }
+
+  /**
+   * Set true to force use of helper stored procedures for migrations.
+   */
+  public void setUseMigrationStoredProcedures(boolean useMigrationStoredProcedures) {
+    this.useMigrationStoredProcedures = useMigrationStoredProcedures;
+  }
+
  /**
   * Return true if Postgres FOR UPDATE should use the NO KEY option.
   */
@@ -314,6 +331,7 @@ public class PlatformConfig {
    databaseBooleanFalse = p.get("databaseBooleanFalse", databaseBooleanFalse);
    databaseInetAddressVarchar = p.getBoolean("databaseInetAddressVarchar", databaseInetAddressVarchar);
    caseSensitiveCollation = p.getBoolean("caseSensitiveCollation", caseSensitiveCollation);
+    useMigrationStoredProcedures = p.getBoolean("useMigrationStoredProcedures", useMigrationStoredProcedures);

    DbUuid dbUuid = p.getEnum(DbUuid.class, "dbuuid", null);
    if (dbUuid != null) {
@@ -53,6 +53,8 @@ public class DatabasePlatform {

  protected boolean supportsSavepointId = true;

+  protected boolean useMigrationStoredProcedures = false;
+
  /**
   * The behaviour used when ending a read only transaction at read committed isolation level.
   */
@@ -237,6 +239,7 @@ public class DatabasePlatform {
  public void configure(PlatformConfig config) {
    this.sequenceBatchSize = config.getDatabaseSequenceBatchSize();
    this.caseSensitiveCollation = config.isCaseSensitiveCollation();
+    this.useMigrationStoredProcedures = config.isUseMigrationStoredProcedures();
    configureIdType(config.getIdType());
    configure(config, config.isAllQuotedIdentifiers());
  }
@@ -343,6 +346,13 @@ public class DatabasePlatform {
    return supportsSavepointId;
  }

+  /**
+   * Return true if migrations should use stored procedures.
+   */
+  public boolean isUseMigrationStoredProcedures() {
+    return useMigrationStoredProcedures;
+  }
+
  /**
   * Return true if the platform supports LIMIT with sql update.
   */
@@ -573,6 +583,10 @@ public class DatabasePlatform {
    this.supportsResultSetConcurrencyModeUpdatable = supportsResultSetConcurrencyModeUpdatable;
  }

+  public void setUseMigrationStoredProcedures(final boolean useMigrationStoredProcedures) {
+    this.useMigrationStoredProcedures = useMigrationStoredProcedures;
+  }
+
  /**
   * Normally not needed - overridden in CockroachPlatform.
   */
@@ -119,4 +119,9 @@ abstract class SqlServerBasePlatform extends DatabasePlatform {
    // for update are hints on from clause of base table
    return sql;
  }
+
+  @Override
+  public boolean isUseMigrationStoredProcedures() {
+    return true;
+  }
 }
@@ -50,6 +50,7 @@ public class DdlGenerator implements SpiDdlGenerator {
  private final ScriptTransform scriptTransform;
  private final Platform platform;
  private final String platformName;
+  private final boolean useMigrationStoredProcedures;

  private CurrentModel currentModel;
  private String dropAllContent;
@@ -71,9 +72,11 @@ public class DdlGenerator implements SpiDdlGenerator {
      log.warn("DDL can't be run on startup with TenantMode " + config.getTenantMode());
      this.runDdl = false;
      this.ddlAutoCommit = false;
+      this.useMigrationStoredProcedures = false;
    } else {
      this.runDdl = config.isDdlRun();
      this.ddlAutoCommit = databasePlatform.isDdlAutoCommit();
+      this.useMigrationStoredProcedures = config.getDatabasePlatform().isUseMigrationStoredProcedures();
    }
    this.scriptTransform = createScriptTransform(config);
    this.baseDir = initBaseDir();
@@ -187,7 +190,7 @@ public class DdlGenerator implements SpiDdlGenerator {

  protected void runDropSql(Connection connection) throws IOException {
    if (!createOnly) {
-      if (extraDdl && jaxbPresent) {
+      if (extraDdl && jaxbPresent && useMigrationStoredProcedures) {
        String extraApply = ExtraDdlXmlReader.buildExtra(platform, true);
        if (extraApply != null) {
          runScript(connection, false, extraApply, "extra-ddl");
@@ -210,7 +213,7 @@ public class DdlGenerator implements SpiDdlGenerator {
    if (extraDdl && jaxbPresent) {
      if (currentModel().isTablePartitioning()) {
        String extraPartitioning = ExtraDdlXmlReader.buildPartitioning(platform);
-        if (extraPartitioning != null && !extraPartitioning.isEmpty()) {
+        if (extraPartitioning != null && !extraPartitioning.isEmpty()  && useMigrationStoredProcedures) {
          runScript(connection, false, extraPartitioning, "builtin-partitioning-ddl");
        }
      }
@@ -392,19 +392,25 @@ public class DefaultDbMigration implements DbMigration {
  private void generateExtraDdl(File migrationDir, DatabasePlatform dbPlatform, boolean tablePartitioning) throws IOException {
    if (dbPlatform != null) {
      if (tablePartitioning && includeBuiltInPartitioning) {
-        generateExtraDdlFor(migrationDir, dbPlatform, ExtraDdlXmlReader.readBuiltinTablePartitioning());
+        generateExtraDdlFor(migrationDir, dbPlatform, ExtraDdlXmlReader.readBuiltinTablePartitioning(), true);
      }
-      generateExtraDdlFor(migrationDir, dbPlatform, ExtraDdlXmlReader.readBuiltin());
-      generateExtraDdlFor(migrationDir, dbPlatform, ExtraDdlXmlReader.read());
+      generateExtraDdlFor(migrationDir, dbPlatform, ExtraDdlXmlReader.readBuiltin(), true);
+      generateExtraDdlFor(migrationDir, dbPlatform, ExtraDdlXmlReader.read(), false);
    }
  }

-  private void generateExtraDdlFor(File migrationDir, DatabasePlatform dbPlatform, ExtraDdl extraDdl) throws IOException {
+  private void generateExtraDdlFor(File migrationDir, DatabasePlatform dbPlatform, ExtraDdl extraDdl, boolean isBuiltin) throws IOException {
    if (extraDdl != null) {
      List<DdlScript> ddlScript = extraDdl.getDdlScript();
      for (DdlScript script : ddlScript) {
        if (!script.isDrop() && matchPlatform(dbPlatform.getPlatform(), script.getPlatforms())) {
-          writeExtraDdl(migrationDir, script);
+          if (script.isInit()) {
+            if (!isBuiltin || dbPlatform.isUseMigrationStoredProcedures()) {
+              writeExtraDdl(migrationDir, script);
+            }
+          } else {
+            writeExtraDdl(migrationDir, script);
+          }
        }
      }
    }
@@ -18,12 +18,15 @@ public class MySqlDdl extends PlatformDdl {
  // this flag is for compatibility. Use it with care.
  private static final boolean USE_CHECK_CONSTRAINT = Boolean.getBoolean("ebean.mysql.useCheckConstraint");

+  private final boolean useMigrationStoredProcedures;
+
  public MySqlDdl(DatabasePlatform platform) {
    super(platform);
    this.alterColumn = "modify";
    this.dropUniqueConstraint = "drop index";
    this.historyDdl = new MySqlHistoryDdl();
    this.inlineComments = true;
+    this.useMigrationStoredProcedures = platform.isUseMigrationStoredProcedures();
  }

  /**
@@ -34,6 +37,15 @@ public class MySqlDdl extends PlatformDdl {
    return "drop index " + maxConstraintName(indexName) + " on " + tableName;
  }

+  @Override
+  public void alterTableDropColumn(final DdlBuffer buffer, final String tableName, final String columnName) throws IOException {
+    if (this.useMigrationStoredProcedures) {
+      buffer.append("CALL usp_ebean_drop_column('").append(tableName).append("', '").append(columnName).append("')").endOfStatement();
+    } else {
+      super.alterTableDropColumn(buffer, tableName, columnName);
+    }
+  }
+
  /**
   * Return the drop foreign key clause.
   */
@@ -112,5 +112,88 @@ BEGIN
 END
 GO
 </ddl-script>
+<ddl-script name="create procs" platforms="mysql" init="true">-- Inital script to create stored procedures etc for mysql platform
+DROP PROCEDURE IF EXISTS usp_ebean_drop_foreign_keys;

+delimiter $$
+--
+-- PROCEDURE: usp_ebean_drop_foreign_keys TABLE, COLUMN
+-- deletes all constraints and foreign keys referring to TABLE.COLUMN
+--
+CREATE PROCEDURE usp_ebean_drop_foreign_keys(IN p_table_name VARCHAR(255), IN p_column_name VARCHAR(255))
+BEGIN
+DECLARE done INT DEFAULT FALSE;
+DECLARE c_fk_name CHAR(255);
+DECLARE curs CURSOR FOR SELECT CONSTRAINT_NAME from information_schema.KEY_COLUMN_USAGE
+WHERE TABLE_SCHEMA = DATABASE() and TABLE_NAME = p_table_name and COLUMN_NAME = p_column_name
+AND REFERENCED_TABLE_NAME IS NOT NULL;
+DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE;
+
+OPEN curs;
+
+read_loop: LOOP
+FETCH curs INTO c_fk_name;
+IF done THEN
+LEAVE read_loop;
+END IF;
+SET @sql = CONCAT('ALTER TABLE ', p_table_name, ' DROP FOREIGN KEY ', c_fk_name);
+PREPARE stmt FROM @sql;
+EXECUTE stmt;
+END LOOP;
+
+CLOSE curs;
+END
+$$
+
+DROP PROCEDURE IF EXISTS usp_ebean_drop_column;
+
+delimiter $$
+--
+-- PROCEDURE: usp_ebean_drop_column TABLE, COLUMN
+-- deletes the column and ensures that all indices and constraints are dropped first
+--
+CREATE PROCEDURE usp_ebean_drop_column(IN p_table_name VARCHAR(255), IN p_column_name VARCHAR(255))
+BEGIN
+CALL usp_ebean_drop_foreign_keys(p_table_name, p_column_name);
+SET @sql = CONCAT('ALTER TABLE ', p_table_name, ' DROP COLUMN ', p_column_name);
+PREPARE stmt FROM @sql;
+EXECUTE stmt;
+END
+$$
+</ddl-script>
+
+<ddl-script name="create procs" platforms="hana" init="true">-- Inital script to create stored procedures etc for the hana platform
+delimiter $$
+--
+-- PROCEDURE: usp_ebean_drop_foreign_keys TABLE, COLUMN
+-- deletes all constraints and foreign keys referring to TABLE.COLUMN
+--
+CREATE OR REPLACE PROCEDURE usp_ebean_drop_foreign_keys(IN table_name NVARCHAR(256), IN column_name NVARCHAR(256))
+AS
+BEGIN
+DECLARE foreign_key_names TABLE(CONSTRAINT_NAME NVARCHAR(256), TABLE_NAME NVARCHAR(256));
+DECLARE i INT;
+
+foreign_key_names = SELECT CONSTRAINT_NAME, TABLE_NAME FROM SYS.REFERENTIAL_CONSTRAINTS WHERE SCHEMA_NAME=CURRENT_SCHEMA AND TABLE_NAME=UPPER(:table_name) AND COLUMN_NAME=UPPER(:column_name);
+
+FOR I IN 1 .. RECORD_COUNT(:foreign_key_names) DO
+EXEC 'ALTER TABLE "' || ESCAPE_DOUBLE_QUOTES(:foreign_key_names.TABLE_NAME[i]) || '" DROP CONSTRAINT "' || ESCAPE_DOUBLE_QUOTES(:foreign_key_names.CONSTRAINT_NAME[i]) || '"';
+END FOR;
+
+END;
+$$
+
+delimiter $$
+--
+-- PROCEDURE: usp_ebean_drop_column TABLE, COLUMN
+-- deletes the column and ensures that all indices and constraints are dropped first
+--
+CREATE OR REPLACE PROCEDURE usp_ebean_drop_column(IN table_name NVARCHAR(256), IN column_name NVARCHAR(256))
+AS
+BEGIN
+CALL usp_ebean_drop_foreign_keys(table_name, column_name);
+EXEC 'ALTER TABLE "' || UPPER(ESCAPE_DOUBLE_QUOTES(table_name)) || '" DROP ("' || UPPER(ESCAPE_DOUBLE_QUOTES(column_name)) || '")';
+END;
+$$
+</ddl-script>
 </extra-ddl>
@@ -0,0 +1,78 @@
+package io.ebeaninternal.dbmigration;
+
+import io.ebean.Database;
+import io.ebean.DatabaseFactory;
+import io.ebean.annotation.Platform;
+import io.ebean.config.DatabaseConfig;
+import io.ebean.config.PlatformConfig;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Class to test the alternative drop behaviour using stored procedures for MySql databases .
+ *
+ * @author Jonas P&ouml;hler, FOCONIS AG
+ */
+public class MysqlGenerateMigrationTest {
+
+  @Test
+  public void testMysqlStoredProcedures() throws Exception {
+    DefaultDbMigration migration = new DefaultDbMigration();
+    migration.setIncludeIndex(true);
+    // We use src/test/resources as output directory (so we see in GIT if files will change)
+    migration.setPathToResources("src/test/resources");
+
+    migration.addPlatform(Platform.MYSQL, "mysql");
+
+    final PlatformConfig platformConfig = new PlatformConfig();
+    platformConfig.setUseMigrationStoredProcedures(true);
+
+    DatabaseConfig config = new DatabaseConfig();
+    config.setName("migrationtest");
+    config.loadFromProperties();
+    config.setPlatformConfig(platformConfig);
+    config.setRegister(false);
+    config.setDefaultServer(false);
+    config.getProperties().put("ebean.migration.migrationPath", "db/migration/mysql");
+
+    config.setPackages(Arrays.asList("misc.migration.mysql_v1_0"));
+    Database server = DatabaseFactory.create(config);
+    migration.setServer(server);
+    migration.setMigrationPath("mysql/procedures");
+
+    // First, we clean up the output-directory
+    assertThat(migration.migrationDirectory().getAbsolutePath()).contains("procedures");
+    Files.walk(migration.migrationDirectory().toPath())
+      .filter(Files::isRegularFile)
+      .map(Path::toFile).forEach(File::delete);
+
+    // then we generate migration scripts for v1_0
+    assertThat(migration.generateMigration()).isEqualTo("1.0__initial");
+
+    config.setPackages(Arrays.asList("misc.migration.mysql_v1_1"));
+    server.shutdown();
+    server = DatabaseFactory.create(config);
+    migration.setServer(server);
+    migration.setMigrationPath("mysql/procedures");
+    assertThat(migration.generateMigration()).isEqualTo("1.1");
+
+    System.setProperty("ddl.migration.pendingDropsFor", "1.1");
+    assertThat(migration.generateMigration()).isEqualTo("1.2__dropsFor_1.1");
+
+    final Path sqlFile = migration.migrationDirectory().toPath()
+      .resolve("mysql/1.2__dropsFor_1.1.sql");
+
+    assertThat(sqlFile).isNotEmptyFile();
+    assertThat(Files.readAllLines(sqlFile, StandardCharsets.UTF_8))
+      .contains("CALL usp_ebean_drop_column('migtest_e_basic', 'status2');")
+      .contains("CALL usp_ebean_drop_column('migtest_e_basic', 'description');");
+
+  }
+}
@@ -0,0 +1,89 @@
+package misc.migration.mysql_v1_0;
+
+import io.ebean.annotation.DbDefault;
+import io.ebean.annotation.EnumValue;
+import io.ebean.annotation.NotNull;
+
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.validation.constraints.Size;
+
+@Entity
+@Table(name = "migtest_e_basic")
+public class EBasic {
+
+  public enum Status {
+    @EnumValue("N")
+    NEW,
+
+    @EnumValue("A")
+    ACTIVE,
+
+    @EnumValue("I")
+    INACTIVE,
+  }
+
+  @Id
+  Integer id;
+
+  Status status;
+
+  @DbDefault("N")
+  @NotNull
+  Status status2;
+
+  @Size(max=127)
+  String name;
+
+  @Size(max=127)
+  String description;
+
+  public EBasic() {
+
+  }
+
+  public EBasic(String name) {
+    this.name = name;
+  }
+
+  public Integer getId() {
+    return id;
+  }
+
+  public void setId(Integer id) {
+    this.id = id;
+  }
+
+  public Status getStatus() {
+    return status;
+  }
+
+  public void setStatus(Status status) {
+    this.status = status;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public String getDescription() {
+    return description;
+  }
+
+  public void setDescription(String description) {
+    this.description = description;
+  }
+
+  public Status getStatus2() {
+    return status2;
+  }
+
+  public void setStatus2(final Status status2) {
+    this.status2 = status2;
+  }
+}
@@ -0,0 +1,64 @@
+package misc.migration.mysql_v1_1;
+
+import io.ebean.annotation.EnumValue;
+
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.validation.constraints.Size;
+
+@Entity
+@Table(name = "migtest_e_basic")
+public class EBasic {
+
+  public enum Status {
+    @EnumValue("N")
+    NEW,
+
+    @EnumValue("A")
+    ACTIVE,
+
+    @EnumValue("I")
+    INACTIVE,
+  }
+
+  @Id
+  Integer id;
+
+  Status status;
+
+  @Size(max=127)
+  String name;
+
+  public EBasic() {
+
+  }
+
+  public EBasic(String name) {
+    this.name = name;
+  }
+
+  public Integer getId() {
+    return id;
+  }
+
+  public void setId(Integer id) {
+    this.id = id;
+  }
+
+  public Status getStatus() {
+    return status;
+  }
+
+  public void setStatus(Status status) {
+    this.status = status;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+}
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<migration xmlns="http://ebean-orm.github.io/xml/ns/dbmigration">
+    <changeSet type="apply">
+        <createTable name="migtest_e_basic" pkName="pk_migtest_e_basic">
+            <column name="id" type="integer" primaryKey="true"/>
+            <column name="status" type="varchar(1)" checkConstraint="check ( status in ('N','A','I'))" checkConstraintName="ck_migtest_e_basic_status"/>
+            <column name="status2" type="varchar(1)" defaultValue="'N'" notnull="true" checkConstraint="check ( status2 in ('N','A','I'))" checkConstraintName="ck_migtest_e_basic_status2"/>
+            <column name="name" type="varchar(127)"/>
+            <column name="description" type="varchar(127)"/>
+        </createTable>
+    </changeSet>
+</migration>
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<migration xmlns="http://ebean-orm.github.io/xml/ns/dbmigration">
+    <changeSet type="pendingDrops">
+        <dropColumn columnName="status2" tableName="migtest_e_basic"/>
+        <dropColumn columnName="description" tableName="migtest_e_basic"/>
+    </changeSet>
+</migration>
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<migration xmlns="http://ebean-orm.github.io/xml/ns/dbmigration">
+    <changeSet type="apply" dropsFor="1.1">
+        <dropColumn columnName="status2" tableName="migtest_e_basic"/>
+        <dropColumn columnName="description" tableName="migtest_e_basic"/>
+    </changeSet>
+</migration>
@@ -0,0 +1,11 @@
+-- Migrationscripts for ebean unittest
+-- apply changes
+create table migtest_e_basic (
+  id                            integer auto_increment not null,
+  status                        varchar(1),
+  status2                       varchar(1) default 'N' not null,
+  name                          varchar(127),
+  description                   varchar(127),
+  constraint pk_migtest_e_basic primary key (id)
+);
+
@@ -0,0 +1,6 @@
+-- Migrationscripts for ebean unittest
+-- apply changes
+CALL usp_ebean_drop_column('migtest_e_basic', 'status2');
+
+CALL usp_ebean_drop_column('migtest_e_basic', 'description');
+
@@ -0,0 +1,48 @@
+-- Inital script to create stored procedures etc for mysql platform
+DROP PROCEDURE IF EXISTS usp_ebean_drop_foreign_keys;
+
+delimiter $$
+--
+-- PROCEDURE: usp_ebean_drop_foreign_keys TABLE, COLUMN
+-- deletes all constraints and foreign keys referring to TABLE.COLUMN
+--
+CREATE PROCEDURE usp_ebean_drop_foreign_keys(IN p_table_name VARCHAR(255), IN p_column_name VARCHAR(255))
+BEGIN
+DECLARE done INT DEFAULT FALSE;
+DECLARE c_fk_name CHAR(255);
+DECLARE curs CURSOR FOR SELECT CONSTRAINT_NAME from information_schema.KEY_COLUMN_USAGE
+WHERE TABLE_SCHEMA = DATABASE() and TABLE_NAME = p_table_name and COLUMN_NAME = p_column_name
+AND REFERENCED_TABLE_NAME IS NOT NULL;
+DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE;
+
+OPEN curs;
+
+read_loop: LOOP
+FETCH curs INTO c_fk_name;
+IF done THEN
+LEAVE read_loop;
+END IF;
+SET @sql = CONCAT('ALTER TABLE ', p_table_name, ' DROP FOREIGN KEY ', c_fk_name);
+PREPARE stmt FROM @sql;
+EXECUTE stmt;
+END LOOP;
+
+CLOSE curs;
+END
+$$
+
+DROP PROCEDURE IF EXISTS usp_ebean_drop_column;
+
+delimiter $$
+--
+-- PROCEDURE: usp_ebean_drop_column TABLE, COLUMN
+-- deletes the column and ensures that all indices and constraints are dropped first
+--
+CREATE PROCEDURE usp_ebean_drop_column(IN p_table_name VARCHAR(255), IN p_column_name VARCHAR(255))
+BEGIN
+CALL usp_ebean_drop_foreign_keys(p_table_name, p_column_name);
+SET @sql = CONCAT('ALTER TABLE ', p_table_name, ' DROP COLUMN ', p_column_name);
+PREPARE stmt FROM @sql;
+EXECUTE stmt;
+END
+$$
@@ -0,0 +1,4 @@
+1835064798,     I__create_procs.sql
+1968521526,     1.0__initial.sql
+-728933533,     1.2__dropsFor_1.1.sql
+