#2284 - ENH: ddl-generator - add optional generation of idx_<platform>.migrations file

ebean-ddl-generator/src/main/java/io/ebean/dbmigration/DbMigration.java

@@ -133,6 +133,14 @@ public interface DbMigration {
    */
   void setStrictMode(boolean strictMode);
 
+  /**
+   * Set to include generation of the index migration file.
+   * <p>
+   * When true this generates a {@code idx_<platform>.migrations} file. This can be used by the migration
+   * runner to improve performance of running migrations, especially when no migration changes have occurred.
+   */
+  void setIncludeIndex(boolean generateIndexFile);
+
   /**
    * Set to true to include a generated header comment in the DDL script.
    */

ebean-ddl-generator/src/main/java/io/ebeaninternal/dbmigration/DefaultDbMigration.java

@@ -106,6 +106,7 @@ public class DefaultDbMigration implements DbMigration {
   private boolean addForeignKeySkipCheck;
   private int lockTimeoutSeconds;
   protected boolean includeBuiltInPartitioning = true;
+  protected boolean includeIndex;
 
   /**
    * Create for offline migration generation.
@@ -182,6 +183,11 @@ public class DefaultDbMigration implements DbMigration {
     this.generatePendingDrop = generatePendingDrop;
   }
 
+  @Override
+  public void setIncludeIndex(boolean includeIndex) {
+    this.includeIndex = includeIndex;
+  }
+
   @Override
   public void setIncludeGeneratedFileComment(boolean includeGeneratedFileComment) {
     this.includeGeneratedFileComment = includeGeneratedFileComment;
@@ -273,7 +279,26 @@ public class DefaultDbMigration implements DbMigration {
    */
   @Override
   public String generateMigration() throws IOException {
-    return generateMigrationFor(false);
+    final String version = generateMigrationFor(false);
+    if (includeIndex) {
+      generateIndex(version);
+    }
+    return version;
+  }
+
+  /**
+   * Generate the {@code idx_platform.migrations} file.
+   */
+  private void generateIndex(String version) throws IOException {
+    final boolean overwrite = version != null;
+    final File topDir = migrationDirectory(false);
+    if (!platforms.isEmpty()) {
+      for (Pair pair : platforms) {
+        new IndexMigration(topDir, pair).generate(overwrite);
+      }
+    } else {
+      new IndexMigration(topDir, databasePlatform).generate(overwrite);
+    }
   }
 
   @Override

ebean-ddl-generator/src/main/java/io/ebeaninternal/dbmigration/IndexMigration.java

@@ -0,0 +1,125 @@
+package io.ebeaninternal.dbmigration;
+
+import io.ebean.config.dbplatform.DatabasePlatform;
+import io.ebean.migration.MigrationVersion;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Stack;
+
+/**
+ * Generate a migration index file.
+ * <p>
+ * This is a file that has all the migrations listed in order with checksum of the file content.
+ */
+class IndexMigration {
+
+  private static final String eol = "\n";
+  private final List<Entry> all = new ArrayList<>();
+  private final File topDir;
+  private final DatabasePlatform databasePlatform;
+  private final File indexFile;
+  private final Stack<String> pathStack = new Stack<>();
+
+  IndexMigration(File topDir, DatabasePlatform databasePlatform) {
+    this.topDir = topDir;
+    this.databasePlatform = databasePlatform;
+    this.indexFile = init();
+  }
+
+  IndexMigration(File topDir, DefaultDbMigration.Pair pair) {
+    this.topDir = new File(topDir, pair.prefix);
+    this.databasePlatform = pair.platform;
+    this.indexFile = init();
+  }
+
+  File init() {
+    pathStack.push("");
+    String name = "idx_" + databasePlatform.getPlatform().base().name().toLowerCase() + ".migrations";
+    return new File(topDir, name);
+  }
+
+  void generate(boolean overwrite) throws IOException {
+    if (!overwrite && indexFile.exists()) {
+      return;
+    }
+    readSqlFiles(topDir);
+    generateIndex();
+  }
+
+  private void generateIndex() throws IOException {
+    Collections.sort(all);
+    FileWriter writer = new FileWriter(indexFile);
+    for (Entry entry : all) {
+      writeChecksumPadded(writer, entry.checksum);
+      writer.write(entry.fileName);
+      writer.write(eol);
+    }
+    writer.write(eol);
+    writer.close();
+  }
+
+  private void writeChecksumPadded(FileWriter writer, int checksum) throws IOException {
+    final String asStr = String.valueOf(checksum);
+    writer.write(asStr);
+    writer.write(',');
+    int max = 15 - asStr.length();
+    for (int i = 0; i < max; i++) {
+      writer.write(' ');
+    }
+  }
+
+  private void readSqlFiles(File dir) {
+    final File[] files = dir.listFiles();
+    if (files != null && files.length > 0) {
+      for (File file : files) {
+        if (file.isDirectory()) {
+          readDirectory(file);
+        }
+        final String lowerName = file.getName().toLowerCase();
+        if (lowerName.endsWith(".sql")) {
+          addEntry(file);
+        }
+      }
+    }
+  }
+
+  private void readDirectory(File dir) {
+    final String current = pathStack.peek();
+    pathStack.push(current + dir.getName() + "/");
+    readSqlFiles(dir);
+    pathStack.pop();
+  }
+
+  private void addEntry(File sqlFile) {
+    final String relativePath = pathStack.peek();
+    final String fileName = sqlFile.getName();
+    final String name = fileName.substring(0, fileName.length() - 4);
+    final MigrationVersion version = MigrationVersion.parse(name);
+    final int checksum = MChecksum.calculate(sqlFile);
+    all.add(new Entry(checksum, version, relativePath + fileName));
+  }
+
+  static class Entry implements Comparable<Entry> {
+
+    private final int checksum;
+    private final String fileName;
+    private final MigrationVersion version;
+
+    Entry(int checksum, MigrationVersion version, String fileName) {
+      this.checksum = checksum;
+      this.version = version;
+      this.fileName = fileName;
+    }
+
+    @Override
+    public int compareTo(Entry other) {
+      return version.compareTo(other.version);
+    }
+  }
+
+}

ebean-ddl-generator/src/main/java/io/ebeaninternal/dbmigration/MChecksum.java

@@ -0,0 +1,29 @@
+package io.ebeaninternal.dbmigration;
+
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.util.zip.CRC32;
+
+/**
+ * Calculates the checksum for the given file content.
+ */
+class MChecksum {
+
+  /**
+   * Returns the checksum of the file. Agnostic of encoding and new line character.
+   */
+  static int calculate(File file) {
+    try {
+      final CRC32 crc32 = new CRC32();
+      BufferedReader bufferedReader = new BufferedReader(new FileReader(file));
+      String line;
+      while ((line = bufferedReader.readLine()) != null) {
+        final byte[] lineBytes = line.getBytes(StandardCharsets.UTF_8);
+        crc32.update(lineBytes, 0, lineBytes.length);
+      }
+      return (int) crc32.getValue();
+    } catch (IOException e) {
+      throw new UncheckedIOException("Failed to calculate checksum", e);
+    }
+  }
+}

ebean-ddl-generator/src/test/java/io/ebeaninternal/dbmigration/DbMigrationGenerateTest.java

@@ -27,20 +27,22 @@ public class DbMigrationGenerateTest {
 
   private static final Logger logger = LoggerFactory.getLogger(DbMigrationGenerateTest.class);
 
-  @Test
-  public void invokeTest() throws IOException {
-    main(null);
+  public static void main(String[] args) throws IOException {
+    run("ebean-ddl-generator/src/test/resources");
   }
 
-  public static void main(String[] args) throws IOException {
+  @Test
+  public void invokeTest() throws IOException {
+    run("src/test/resources");
+  }
 
-    logger.info("start");
+  public static void run(String pathToResources) throws IOException {
+    logger.info("start current directory: " + new File(".").getAbsolutePath());
 
     DefaultDbMigration migration = new DefaultDbMigration();
-
+    migration.setIncludeIndex(true);
     // We use src/test/resources as output directory (so we see in GIT if files will change)
-
-    migration.setPathToResources("src/test/resources");
+    migration.setPathToResources(pathToResources);
     migration.setMigrationPath("db/migration");
     migration.setMigrationPath(null); // use the default for this test
 

ebean-ddl-generator/src/test/java/io/ebeaninternal/dbmigration/IndexMigrationTest.java

@@ -0,0 +1,67 @@
+package io.ebeaninternal.dbmigration;
+
+import io.ebean.config.dbplatform.DatabasePlatform;
+import io.ebean.config.dbplatform.h2.H2Platform;
+import io.ebean.config.dbplatform.postgres.PostgresPlatform;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class IndexMigrationTest {
+
+  @Test
+  public void index() throws IOException {
+    File topDir = new File("src/test/resources/dbmigration/index");
+    if (!topDir.exists()) {
+      throw new IllegalStateException("Not expected - dir does not exist " + topDir.getAbsolutePath());
+    }
+    DatabasePlatform pg = new PostgresPlatform();
+    IndexMigration indexMigration = new IndexMigration(topDir, pg);
+    indexMigration.generate(true);
+
+
+    File expected = new File(topDir, "idx_postgres.migrations");
+    assertThat(expected).exists();
+
+    final List<String> expectedLines = Arrays.asList(
+      "-965417868,     I__init_1.sql",
+      "907060870,      1.0__hello.sql",
+      "-1938594527,    1.1__foo.sql",
+      "-1960070312,    R__view_1.sql");
+
+    final List<String> lines = Files.readAllLines(expected.toPath(), StandardCharsets.UTF_8);
+    assertThat(lines).containsAll(expectedLines);
+  }
+
+  @Test
+  public void index2_withSubDirectories() throws IOException {
+    File topDir = new File("src/test/resources/dbmigration/index2");
+    if (!topDir.exists()) {
+      throw new IllegalStateException("Not expected - dir does not exist " + topDir.getAbsolutePath());
+    }
+    DatabasePlatform pg = new H2Platform();
+    IndexMigration indexMigration = new IndexMigration(topDir, pg);
+    indexMigration.generate(true);
+
+    File expected = new File(topDir, "idx_h2.migrations");
+    assertThat(expected).exists();
+
+    final List<String> expectedLines = Arrays.asList(
+      "-965417868,     I__init_1.sql",
+      "-390611389,     g1/1.0__a.sql",
+      "1908338681,     g1/1.1__b.sql",
+      "-1776543936,    g2/2.0__a.sql",
+      "253052666,      g2/2.1__2b.sql",
+      "-1960070312,    R__view_1.sql");
+
+    final List<String> lines = Files.readAllLines(expected.toPath(), StandardCharsets.UTF_8);
+    assertThat(lines).containsAll(expectedLines);
+  }
+}

ebean-ddl-generator/src/test/java/io/ebeaninternal/dbmigration/MChecksumTest.java

@@ -0,0 +1,18 @@
+package io.ebeaninternal.dbmigration;
+
+import org.junit.Test;
+
+import java.io.File;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class MChecksumTest {
+
+  @Test
+  public void calculate() {
+    File file = new File("src/test/resources/dbmigration/index/1.0__hello.sql");
+    assertThat(file).exists();
+
+    assertThat(MChecksum.calculate(file)).isEqualTo(907060870);
+  }
+}

ebean-ddl-generator/src/test/resources/dbmigration/index/1.0__hello.sql

@@ -0,0 +1 @@
+hello

ebean-ddl-generator/src/test/resources/dbmigration/index/1.1__foo.sql

@@ -0,0 +1 @@
+foo

ebean-ddl-generator/src/test/resources/dbmigration/index/I__init_1.sql

@@ -0,0 +1 @@
+init

ebean-ddl-generator/src/test/resources/dbmigration/index/R__view_1.sql

@@ -0,0 +1 @@
+Something

ebean-ddl-generator/src/test/resources/dbmigration/index2/I__init_1.sql

@@ -0,0 +1 @@
+init

ebean-ddl-generator/src/test/resources/dbmigration/index2/R__view_1.sql

@@ -0,0 +1 @@
+Something

ebean-ddl-generator/src/test/resources/dbmigration/index2/g1/1.0__a.sql

@@ -0,0 +1 @@
+a

ebean-ddl-generator/src/test/resources/dbmigration/index2/g1/1.1__b.sql

@@ -0,0 +1 @@
+b

ebean-ddl-generator/src/test/resources/dbmigration/index2/g2/2.0__a.sql

@@ -0,0 +1 @@
+2a

ebean-ddl-generator/src/test/resources/dbmigration/index2/g2/2.1__2b.sql

@@ -0,0 +1 @@
+2b