diff --git a/ebean-ddl-generator/src/main/java/io/ebean/dbmigration/DbMigration.java b/ebean-ddl-generator/src/main/java/io/ebean/dbmigration/DbMigration.java index 55753f4a2..ec6db4597 100644 --- a/ebean-ddl-generator/src/main/java/io/ebean/dbmigration/DbMigration.java +++ b/ebean-ddl-generator/src/main/java/io/ebean/dbmigration/DbMigration.java @@ -133,6 +133,14 @@ public interface DbMigration { */ void setStrictMode(boolean strictMode); + /** + * Set to include generation of the index migration file. + *

+ * When true this generates a {@code idx_.migrations} file. This can be used by the migration + * runner to improve performance of running migrations, especially when no migration changes have occurred. + */ + void setIncludeIndex(boolean generateIndexFile); + /** * Set to true to include a generated header comment in the DDL script. */ diff --git a/ebean-ddl-generator/src/main/java/io/ebeaninternal/dbmigration/DefaultDbMigration.java b/ebean-ddl-generator/src/main/java/io/ebeaninternal/dbmigration/DefaultDbMigration.java index 7643eeec2..7f2a90558 100644 --- a/ebean-ddl-generator/src/main/java/io/ebeaninternal/dbmigration/DefaultDbMigration.java +++ b/ebean-ddl-generator/src/main/java/io/ebeaninternal/dbmigration/DefaultDbMigration.java @@ -106,6 +106,7 @@ public class DefaultDbMigration implements DbMigration { private boolean addForeignKeySkipCheck; private int lockTimeoutSeconds; protected boolean includeBuiltInPartitioning = true; + protected boolean includeIndex; /** * Create for offline migration generation. @@ -182,6 +183,11 @@ public class DefaultDbMigration implements DbMigration { this.generatePendingDrop = generatePendingDrop; } + @Override + public void setIncludeIndex(boolean includeIndex) { + this.includeIndex = includeIndex; + } + @Override public void setIncludeGeneratedFileComment(boolean includeGeneratedFileComment) { this.includeGeneratedFileComment = includeGeneratedFileComment; @@ -273,7 +279,26 @@ public class DefaultDbMigration implements DbMigration { */ @Override public String generateMigration() throws IOException { - return generateMigrationFor(false); + final String version = generateMigrationFor(false); + if (includeIndex) { + generateIndex(version); + } + return version; + } + + /** + * Generate the {@code idx_platform.migrations} file. + */ + private void generateIndex(String version) throws IOException { + final boolean overwrite = version != null; + final File topDir = migrationDirectory(false); + if (!platforms.isEmpty()) { + for (Pair pair : platforms) { + new IndexMigration(topDir, pair).generate(overwrite); + } + } else { + new IndexMigration(topDir, databasePlatform).generate(overwrite); + } } @Override diff --git a/ebean-ddl-generator/src/main/java/io/ebeaninternal/dbmigration/IndexMigration.java b/ebean-ddl-generator/src/main/java/io/ebeaninternal/dbmigration/IndexMigration.java new file mode 100644 index 000000000..215c18e72 --- /dev/null +++ b/ebean-ddl-generator/src/main/java/io/ebeaninternal/dbmigration/IndexMigration.java @@ -0,0 +1,125 @@ +package io.ebeaninternal.dbmigration; + +import io.ebean.config.dbplatform.DatabasePlatform; +import io.ebean.migration.MigrationVersion; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Stack; + +/** + * Generate a migration index file. + *

+ * This is a file that has all the migrations listed in order with checksum of the file content. + */ +class IndexMigration { + + private static final String eol = "\n"; + private final List all = new ArrayList<>(); + private final File topDir; + private final DatabasePlatform databasePlatform; + private final File indexFile; + private final Stack pathStack = new Stack<>(); + + IndexMigration(File topDir, DatabasePlatform databasePlatform) { + this.topDir = topDir; + this.databasePlatform = databasePlatform; + this.indexFile = init(); + } + + IndexMigration(File topDir, DefaultDbMigration.Pair pair) { + this.topDir = new File(topDir, pair.prefix); + this.databasePlatform = pair.platform; + this.indexFile = init(); + } + + File init() { + pathStack.push(""); + String name = "idx_" + databasePlatform.getPlatform().base().name().toLowerCase() + ".migrations"; + return new File(topDir, name); + } + + void generate(boolean overwrite) throws IOException { + if (!overwrite && indexFile.exists()) { + return; + } + readSqlFiles(topDir); + generateIndex(); + } + + private void generateIndex() throws IOException { + Collections.sort(all); + FileWriter writer = new FileWriter(indexFile); + for (Entry entry : all) { + writeChecksumPadded(writer, entry.checksum); + writer.write(entry.fileName); + writer.write(eol); + } + writer.write(eol); + writer.close(); + } + + private void writeChecksumPadded(FileWriter writer, int checksum) throws IOException { + final String asStr = String.valueOf(checksum); + writer.write(asStr); + writer.write(','); + int max = 15 - asStr.length(); + for (int i = 0; i < max; i++) { + writer.write(' '); + } + } + + private void readSqlFiles(File dir) { + final File[] files = dir.listFiles(); + if (files != null && files.length > 0) { + for (File file : files) { + if (file.isDirectory()) { + readDirectory(file); + } + final String lowerName = file.getName().toLowerCase(); + if (lowerName.endsWith(".sql")) { + addEntry(file); + } + } + } + } + + private void readDirectory(File dir) { + final String current = pathStack.peek(); + pathStack.push(current + dir.getName() + "/"); + readSqlFiles(dir); + pathStack.pop(); + } + + private void addEntry(File sqlFile) { + final String relativePath = pathStack.peek(); + final String fileName = sqlFile.getName(); + final String name = fileName.substring(0, fileName.length() - 4); + final MigrationVersion version = MigrationVersion.parse(name); + final int checksum = MChecksum.calculate(sqlFile); + all.add(new Entry(checksum, version, relativePath + fileName)); + } + + static class Entry implements Comparable { + + private final int checksum; + private final String fileName; + private final MigrationVersion version; + + Entry(int checksum, MigrationVersion version, String fileName) { + this.checksum = checksum; + this.version = version; + this.fileName = fileName; + } + + @Override + public int compareTo(Entry other) { + return version.compareTo(other.version); + } + } + +} diff --git a/ebean-ddl-generator/src/main/java/io/ebeaninternal/dbmigration/MChecksum.java b/ebean-ddl-generator/src/main/java/io/ebeaninternal/dbmigration/MChecksum.java new file mode 100644 index 000000000..9c533992e --- /dev/null +++ b/ebean-ddl-generator/src/main/java/io/ebeaninternal/dbmigration/MChecksum.java @@ -0,0 +1,29 @@ +package io.ebeaninternal.dbmigration; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.zip.CRC32; + +/** + * Calculates the checksum for the given file content. + */ +class MChecksum { + + /** + * Returns the checksum of the file. Agnostic of encoding and new line character. + */ + static int calculate(File file) { + try { + final CRC32 crc32 = new CRC32(); + BufferedReader bufferedReader = new BufferedReader(new FileReader(file)); + String line; + while ((line = bufferedReader.readLine()) != null) { + final byte[] lineBytes = line.getBytes(StandardCharsets.UTF_8); + crc32.update(lineBytes, 0, lineBytes.length); + } + return (int) crc32.getValue(); + } catch (IOException e) { + throw new UncheckedIOException("Failed to calculate checksum", e); + } + } +} diff --git a/ebean-ddl-generator/src/test/java/io/ebeaninternal/dbmigration/DbMigrationGenerateTest.java b/ebean-ddl-generator/src/test/java/io/ebeaninternal/dbmigration/DbMigrationGenerateTest.java index befead534..f5bf01a56 100644 --- a/ebean-ddl-generator/src/test/java/io/ebeaninternal/dbmigration/DbMigrationGenerateTest.java +++ b/ebean-ddl-generator/src/test/java/io/ebeaninternal/dbmigration/DbMigrationGenerateTest.java @@ -27,20 +27,22 @@ public class DbMigrationGenerateTest { private static final Logger logger = LoggerFactory.getLogger(DbMigrationGenerateTest.class); - @Test - public void invokeTest() throws IOException { - main(null); + public static void main(String[] args) throws IOException { + run("ebean-ddl-generator/src/test/resources"); } - public static void main(String[] args) throws IOException { + @Test + public void invokeTest() throws IOException { + run("src/test/resources"); + } - logger.info("start"); + public static void run(String pathToResources) throws IOException { + logger.info("start current directory: " + new File(".").getAbsolutePath()); DefaultDbMigration migration = new DefaultDbMigration(); - + migration.setIncludeIndex(true); // We use src/test/resources as output directory (so we see in GIT if files will change) - - migration.setPathToResources("src/test/resources"); + migration.setPathToResources(pathToResources); migration.setMigrationPath("db/migration"); migration.setMigrationPath(null); // use the default for this test diff --git a/ebean-ddl-generator/src/test/java/io/ebeaninternal/dbmigration/IndexMigrationTest.java b/ebean-ddl-generator/src/test/java/io/ebeaninternal/dbmigration/IndexMigrationTest.java new file mode 100644 index 000000000..5a04d8917 --- /dev/null +++ b/ebean-ddl-generator/src/test/java/io/ebeaninternal/dbmigration/IndexMigrationTest.java @@ -0,0 +1,67 @@ +package io.ebeaninternal.dbmigration; + +import io.ebean.config.dbplatform.DatabasePlatform; +import io.ebean.config.dbplatform.h2.H2Platform; +import io.ebean.config.dbplatform.postgres.PostgresPlatform; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class IndexMigrationTest { + + @Test + public void index() throws IOException { + File topDir = new File("src/test/resources/dbmigration/index"); + if (!topDir.exists()) { + throw new IllegalStateException("Not expected - dir does not exist " + topDir.getAbsolutePath()); + } + DatabasePlatform pg = new PostgresPlatform(); + IndexMigration indexMigration = new IndexMigration(topDir, pg); + indexMigration.generate(true); + + + File expected = new File(topDir, "idx_postgres.migrations"); + assertThat(expected).exists(); + + final List expectedLines = Arrays.asList( + "-965417868, I__init_1.sql", + "907060870, 1.0__hello.sql", + "-1938594527, 1.1__foo.sql", + "-1960070312, R__view_1.sql"); + + final List lines = Files.readAllLines(expected.toPath(), StandardCharsets.UTF_8); + assertThat(lines).containsAll(expectedLines); + } + + @Test + public void index2_withSubDirectories() throws IOException { + File topDir = new File("src/test/resources/dbmigration/index2"); + if (!topDir.exists()) { + throw new IllegalStateException("Not expected - dir does not exist " + topDir.getAbsolutePath()); + } + DatabasePlatform pg = new H2Platform(); + IndexMigration indexMigration = new IndexMigration(topDir, pg); + indexMigration.generate(true); + + File expected = new File(topDir, "idx_h2.migrations"); + assertThat(expected).exists(); + + final List expectedLines = Arrays.asList( + "-965417868, I__init_1.sql", + "-390611389, g1/1.0__a.sql", + "1908338681, g1/1.1__b.sql", + "-1776543936, g2/2.0__a.sql", + "253052666, g2/2.1__2b.sql", + "-1960070312, R__view_1.sql"); + + final List lines = Files.readAllLines(expected.toPath(), StandardCharsets.UTF_8); + assertThat(lines).containsAll(expectedLines); + } +} diff --git a/ebean-ddl-generator/src/test/java/io/ebeaninternal/dbmigration/MChecksumTest.java b/ebean-ddl-generator/src/test/java/io/ebeaninternal/dbmigration/MChecksumTest.java new file mode 100644 index 000000000..e52c43755 --- /dev/null +++ b/ebean-ddl-generator/src/test/java/io/ebeaninternal/dbmigration/MChecksumTest.java @@ -0,0 +1,18 @@ +package io.ebeaninternal.dbmigration; + +import org.junit.Test; + +import java.io.File; + +import static org.assertj.core.api.Assertions.assertThat; + +public class MChecksumTest { + + @Test + public void calculate() { + File file = new File("src/test/resources/dbmigration/index/1.0__hello.sql"); + assertThat(file).exists(); + + assertThat(MChecksum.calculate(file)).isEqualTo(907060870); + } +} diff --git a/ebean-ddl-generator/src/test/resources/dbmigration/index/1.0__hello.sql b/ebean-ddl-generator/src/test/resources/dbmigration/index/1.0__hello.sql new file mode 100644 index 000000000..ce0136250 --- /dev/null +++ b/ebean-ddl-generator/src/test/resources/dbmigration/index/1.0__hello.sql @@ -0,0 +1 @@ +hello diff --git a/ebean-ddl-generator/src/test/resources/dbmigration/index/1.1__foo.sql b/ebean-ddl-generator/src/test/resources/dbmigration/index/1.1__foo.sql new file mode 100644 index 000000000..257cc5642 --- /dev/null +++ b/ebean-ddl-generator/src/test/resources/dbmigration/index/1.1__foo.sql @@ -0,0 +1 @@ +foo diff --git a/ebean-ddl-generator/src/test/resources/dbmigration/index/I__init_1.sql b/ebean-ddl-generator/src/test/resources/dbmigration/index/I__init_1.sql new file mode 100644 index 000000000..b1b716105 --- /dev/null +++ b/ebean-ddl-generator/src/test/resources/dbmigration/index/I__init_1.sql @@ -0,0 +1 @@ +init diff --git a/ebean-ddl-generator/src/test/resources/dbmigration/index/R__view_1.sql b/ebean-ddl-generator/src/test/resources/dbmigration/index/R__view_1.sql new file mode 100644 index 000000000..0f2416ebf --- /dev/null +++ b/ebean-ddl-generator/src/test/resources/dbmigration/index/R__view_1.sql @@ -0,0 +1 @@ +Something diff --git a/ebean-ddl-generator/src/test/resources/dbmigration/index2/I__init_1.sql b/ebean-ddl-generator/src/test/resources/dbmigration/index2/I__init_1.sql new file mode 100644 index 000000000..b1b716105 --- /dev/null +++ b/ebean-ddl-generator/src/test/resources/dbmigration/index2/I__init_1.sql @@ -0,0 +1 @@ +init diff --git a/ebean-ddl-generator/src/test/resources/dbmigration/index2/R__view_1.sql b/ebean-ddl-generator/src/test/resources/dbmigration/index2/R__view_1.sql new file mode 100644 index 000000000..0f2416ebf --- /dev/null +++ b/ebean-ddl-generator/src/test/resources/dbmigration/index2/R__view_1.sql @@ -0,0 +1 @@ +Something diff --git a/ebean-ddl-generator/src/test/resources/dbmigration/index2/g1/1.0__a.sql b/ebean-ddl-generator/src/test/resources/dbmigration/index2/g1/1.0__a.sql new file mode 100644 index 000000000..789819226 --- /dev/null +++ b/ebean-ddl-generator/src/test/resources/dbmigration/index2/g1/1.0__a.sql @@ -0,0 +1 @@ +a diff --git a/ebean-ddl-generator/src/test/resources/dbmigration/index2/g1/1.1__b.sql b/ebean-ddl-generator/src/test/resources/dbmigration/index2/g1/1.1__b.sql new file mode 100644 index 000000000..617807982 --- /dev/null +++ b/ebean-ddl-generator/src/test/resources/dbmigration/index2/g1/1.1__b.sql @@ -0,0 +1 @@ +b diff --git a/ebean-ddl-generator/src/test/resources/dbmigration/index2/g2/2.0__a.sql b/ebean-ddl-generator/src/test/resources/dbmigration/index2/g2/2.0__a.sql new file mode 100644 index 000000000..94226dabb --- /dev/null +++ b/ebean-ddl-generator/src/test/resources/dbmigration/index2/g2/2.0__a.sql @@ -0,0 +1 @@ +2a diff --git a/ebean-ddl-generator/src/test/resources/dbmigration/index2/g2/2.1__2b.sql b/ebean-ddl-generator/src/test/resources/dbmigration/index2/g2/2.1__2b.sql new file mode 100644 index 000000000..b8a4cf4af --- /dev/null +++ b/ebean-ddl-generator/src/test/resources/dbmigration/index2/g2/2.1__2b.sql @@ -0,0 +1 @@ +2b