From 56a742ad7037e2a985638ea4a6d5510545955ecc Mon Sep 17 00:00:00 2001 From: Roland Praml Date: Fri, 2 Mar 2018 00:09:21 +0100 Subject: [PATCH] NEW: UUID version 1 generator (#1309) * NEW: Uuid Type 1 generator * Remove parameter (from other PR) * FIX: Mac detection * Have to use nodeId in generation --- .gitignore | 1 + .../java/io/ebean/config/ServerConfig.java | 48 +++++ .../deploy/meta/DeployBeanDescriptor.java | 22 +- .../server/idgen/UuidV1IdGenerator.java | 197 ++++++++++++++++++ .../server/idgen/UuidV1RndIdGenerator.java | 179 ++++++++++++++++ ...dGenerator.java => UuidV4IdGenerator.java} | 8 +- .../server/idgen/TestUuidGenerator.java | 141 +++++++++++++ 7 files changed, 591 insertions(+), 5 deletions(-) create mode 100644 src/main/java/io/ebeaninternal/server/idgen/UuidV1IdGenerator.java create mode 100644 src/main/java/io/ebeaninternal/server/idgen/UuidV1RndIdGenerator.java rename src/main/java/io/ebeaninternal/server/idgen/{UuidIdGenerator.java => UuidV4IdGenerator.java} (68%) create mode 100644 src/test/java/io/ebeaninternal/server/idgen/TestUuidGenerator.java diff --git a/.gitignore b/.gitignore index 0d96b2f95..1b88dd830 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ profiling/ *.ipr *.iws .idea/ +*uuid.state \ No newline at end of file diff --git a/src/main/java/io/ebean/config/ServerConfig.java b/src/main/java/io/ebean/config/ServerConfig.java index 69a771ef3..0b42551b1 100644 --- a/src/main/java/io/ebean/config/ServerConfig.java +++ b/src/main/java/io/ebean/config/ServerConfig.java @@ -372,6 +372,16 @@ public class ServerConfig { */ private DbTypeConfig dbTypeConfig = new DbTypeConfig(); + /** + * The UUID version to use. + */ + private UuidVersion uuidVersion = UuidVersion.VERSION4; + + /** + * The UUID state file (for Version 1 UUIDs). + */ + private String uuidStateFile = "ebean-uuid.state"; + private List idGenerators = new ArrayList<>(); private List findControllers = new ArrayList<>(); private List persistControllers = new ArrayList<>(); @@ -1891,6 +1901,34 @@ public class ServerConfig { this.dbTypeConfig.setDbUuid(dbUuid); } + /** + * Returns the UUID version mode. + */ + public UuidVersion getUuidVersion() { + return uuidVersion; + } + + /** + * Sets the UUID version mode. + */ + public void setUuidVersion(UuidVersion uuidVersion) { + this.uuidVersion = uuidVersion; + } + + /** + * Return the UUID state file. + */ + public String getUuidStateFile() { + return uuidStateFile; + } + + /** + * Set the UUID state file. + */ + public void setUuidStateFile(String uuidStateFile) { + this.uuidStateFile = uuidStateFile; + } + /** * Return true if LocalTime should be persisted with nanos precision. */ @@ -2816,6 +2854,10 @@ public class ServerConfig { if (p.getBoolean("uuidStoreAsBinary", false)) { dbTypeConfig.setDbUuid(DbUuid.BINARY); } + + uuidVersion = p.getEnum(UuidVersion.class, "uuidVersion", uuidVersion); + uuidStateFile = p.get("uuidStateFile", uuidStateFile); + localTimeWithNanos = p.getBoolean("localTimeWithNanos", localTimeWithNanos); jodaLocalTimeMode = p.get("jodaLocalTimeMode", jodaLocalTimeMode); @@ -3101,4 +3143,10 @@ public class ServerConfig { return binaryOptimized; } } + + public enum UuidVersion { + VERSION4, + VERSION1, + VERSION1RND + }; } diff --git a/src/main/java/io/ebeaninternal/server/deploy/meta/DeployBeanDescriptor.java b/src/main/java/io/ebeaninternal/server/deploy/meta/DeployBeanDescriptor.java index 83416486a..cfd837967 100644 --- a/src/main/java/io/ebeaninternal/server/deploy/meta/DeployBeanDescriptor.java +++ b/src/main/java/io/ebeaninternal/server/deploy/meta/DeployBeanDescriptor.java @@ -29,7 +29,9 @@ import io.ebeaninternal.server.deploy.IndexDefinition; import io.ebeaninternal.server.deploy.InheritInfo; import io.ebeaninternal.server.deploy.TableJoin; import io.ebeaninternal.server.deploy.parse.DeployBeanInfo; -import io.ebeaninternal.server.idgen.UuidIdGenerator; +import io.ebeaninternal.server.idgen.UuidV1IdGenerator; +import io.ebeaninternal.server.idgen.UuidV1RndIdGenerator; +import io.ebeaninternal.server.idgen.UuidV4IdGenerator; import io.ebeaninternal.server.rawsql.SpiRawSql; import javax.persistence.Entity; @@ -844,8 +846,22 @@ public class DeployBeanDescriptor { */ public void setUuidGenerator() { this.idType = IdType.EXTERNAL; - this.idGeneratorName = UuidIdGenerator.AUTO_UUID; - this.idGenerator = UuidIdGenerator.INSTANCE; + this.idGeneratorName = PlatformIdGenerator.AUTO_UUID; + + switch (serverConfig.getUuidVersion()) { + case VERSION1: + this.idGenerator = UuidV1IdGenerator.getInstance(serverConfig.getUuidStateFile()); + break; + + case VERSION1RND: + this.idGenerator = UuidV1RndIdGenerator.INSTANCE; + break; + + case VERSION4: + default: + this.idGenerator = UuidV4IdGenerator.INSTANCE; + break; + } } /** diff --git a/src/main/java/io/ebeaninternal/server/idgen/UuidV1IdGenerator.java b/src/main/java/io/ebeaninternal/server/idgen/UuidV1IdGenerator.java new file mode 100644 index 000000000..0202ab816 --- /dev/null +++ b/src/main/java/io/ebeaninternal/server/idgen/UuidV1IdGenerator.java @@ -0,0 +1,197 @@ +package io.ebeaninternal.server.idgen; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.util.Date; +import java.util.Enumeration; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * IdGenerator for java util UUID. + * + * It extends the UuidV1RndIdGenerator so that it can generate rfc4122 compliant Type 1 UUIDs. + * + * This generator produces real Type 1 UUIDs (best for sqlserver) - You should use this generator only, + * if you can guarantee, that mac addess is uniqe. + */ +public class UuidV1IdGenerator extends UuidV1RndIdGenerator { + + private static final Logger logger = LoggerFactory.getLogger("io.ebean.IDGEN"); + + private final File stateFile; + + private byte[] nodeId = null; + + private boolean canSaveState = true; + + private static final Map INSTANCES = new ConcurrentHashMap<>(); + + /** + * Returns an instance for given file. + */ + public static UuidV1IdGenerator getInstance(String file) { + return INSTANCES.computeIfAbsent(new File(file), UuidV1IdGenerator::new); + } + + /** + * Returns an instance for given file. + */ + public static UuidV1IdGenerator getInstance(File file) { + return INSTANCES.computeIfAbsent(file, UuidV1IdGenerator::new); + } + + /** + * Returns an alternative node id - set with the 'ebean.uuid.nodeId' system property. + */ + private static byte[] getAlternativeNodeId() { + try { + String altNodeId = System.getProperty("ebean.uuid.nodeId"); + if (altNodeId != null) { + String[] components = altNodeId.split("-"); + if (components.length != 5) { + throw new IllegalArgumentException("Invalid nodeId string: " + altNodeId); + } + byte[] nodeId = new byte[6]; + for (int i=0; i<5; i++) { + nodeId[i] = Byte.decode("0x"+components[i]).byteValue(); + } + return nodeId; + } + } catch (SecurityException se) { + // ignore + } + return null; + } + + /** + * Find hardware ID. + */ + private static byte[] getHardwareId() throws SocketException { + final Enumeration e = NetworkInterface.getNetworkInterfaces(); + while (e.hasMoreElements()) { + NetworkInterface network = e.nextElement(); + if (!network.isLoopback()) { + return network.getHardwareAddress(); + } + } + return null; + } + + /** + * Creates a new instance of UuidGenerator. Note that there should not be more + * than one instance per stateFile. + */ + private UuidV1IdGenerator(final File stateFile) { + super(); + this.stateFile = stateFile; + try { + // See, if there is an alternative MAC address set. + nodeId = getAlternativeNodeId(); + if (nodeId != null) { + logger.info("Using alternative MAC {} to generate Type 1 UUIDs", getNodeIdentifier()); + } else { + nodeId = getHardwareId(); + logger.info("Using MAC {} to generate Type 1 UUIDs", getNodeIdentifier()); + } + if (nodeId == null) { + canSaveState = false; + // RFC 4.5 use random portion for node + nodeId = super.getNodeIdBytes(); + logger.error("Have to fall back to random node identifier {} (Reason: No suitable network interface found)", getNodeIdentifier()); + + } else { + boolean flag = restoreState(); + UUID uuid = nextId(null); + long ts = timeStamp.get(); + ts -= UUID_EPOCH_OFFSET; + ts /= MILLIS_TO_UUID; + + saveState(); + logger.debug("RestoreState: {}, ClockSeq {}, Timestamp {}, uuid {}, stateFile: {})", flag, clockSeq.get(), + new Date(ts), uuid, stateFile); + } + } catch (IOException e) { + canSaveState = false; + // RFC 4.5 use random portion for node + nodeId = super.getNodeIdBytes(); + logger.error("Have to fall back to random node identifier {} (Reason: {} )", getNodeIdentifier(), e.getMessage()); + } + } + + /** + * Returns the Node-identifier (=MAC address) as string + */ + public String getNodeIdentifier() { + if (nodeId == null) { + return "none"; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < nodeId.length; i++) { + sb.append(String.format("%02X%s", nodeId[i], (i < nodeId.length - 1) ? "-" : "")); + } + return sb.toString(); + } + + /** + * Restores the state from the state file. + */ + private boolean restoreState() throws IOException { + Properties prop = new Properties(); + if (stateFile.exists()) { + try (InputStream is = new FileInputStream(stateFile)) { + prop.load(is); + } + } + if (getNodeIdentifier().equals(prop.getProperty("nodeId"))) { + try { + Integer seq = Integer.valueOf(prop.getProperty("clockSeq")) & 0x3FFF; + Long ts = Long.valueOf(prop.getProperty("timeStamp")); + clockSeq.set(seq); + timeStamp.set(ts); + logger.debug("Restored state from '{}'", stateFile); + return true; + } catch (NumberFormatException nfe) { + // nop + } + } + return false; + } + + /** + * Saves the state to the state file; + */ + @Override + protected void saveState() { + if (!canSaveState) { + return; + } + Properties prop = new Properties(); + prop.setProperty("nodeId", getNodeIdentifier()); + prop.setProperty("clockSeq", String.valueOf(clockSeq.get())); + prop.setProperty("timeStamp", String.valueOf(timeStamp.get())); + try (OutputStream os = new FileOutputStream(stateFile)) { + prop.store(os, "ebean uuid state file"); + logger.debug("Persisted state to '{}'", stateFile); + } catch (IOException e) { + logger.error("Could not persist uuid state to '{}'", stateFile, e); + } + } + + @Override + protected byte[] getNodeIdBytes() { + return nodeId; + } + +} diff --git a/src/main/java/io/ebeaninternal/server/idgen/UuidV1RndIdGenerator.java b/src/main/java/io/ebeaninternal/server/idgen/UuidV1RndIdGenerator.java new file mode 100644 index 000000000..4db84223d --- /dev/null +++ b/src/main/java/io/ebeaninternal/server/idgen/UuidV1RndIdGenerator.java @@ -0,0 +1,179 @@ +package io.ebeaninternal.server.idgen; + +import io.ebean.Transaction; +import io.ebean.config.dbplatform.PlatformIdGenerator; + +import java.security.SecureRandom; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * IdGenerator for (pseudo) type 1 UUIDs. + * + * This implementation generates a type 1 UUID according to + * https://tools.ietf.org/html/rfc4122.html#section-4.2 + * but has no persistence storage. It generates a new random 47 bit node ID with every + * UUID. + * + * Use this, if you want randomness in your UUIDs but want to take advantage of index + * optimizations of the database. It may be good with AUTO_BINARY_OPTIMIZED and MySql. + * + * See: https://www.percona.com/blog/2014/12/19/store-uuid-optimized-way/ + */ +public class UuidV1RndIdGenerator implements PlatformIdGenerator { + + protected static final Logger logger = LoggerFactory.getLogger("io.ebean.IDGEN"); + + // UUID epoch 1582-10-15 00:00:00 and the Unix epoch 1970-01-01 00:00:00. + protected static final long UUID_EPOCH_OFFSET = 0x01B21DD213814000L; + + // the resolution is 100ns + protected static final long MILLIS_TO_UUID = 10000; + + public static final UuidV1RndIdGenerator INSTANCE = new UuidV1RndIdGenerator(); + + protected final AtomicInteger clockSeq = new AtomicInteger((int) (Math.random() * 0x3FFF)); + + private final SecureRandom numberGenerator = new SecureRandom(); + + protected AtomicLong timeStamp = new AtomicLong(currentUuidTime()); + + private AtomicLong nanoToMilliOffset = new AtomicLong(currentUuidTime()); + + + /** + * Returns the uuid epoch. + * + * This is the number of 100ns intervals since 1582-10-15 00:00:00 + */ + private static long currentUuidTime() { + return (System.currentTimeMillis() * MILLIS_TO_UUID) + UUID_EPOCH_OFFSET; + } + + public UuidV1RndIdGenerator() { + computeNanoOffset(); + } + + /** + * Computes the internal offset between System.nanoTime() and System.currentTimeMillis(). + */ + protected void computeNanoOffset() { + long currentTime = currentUuidTime(); + long fromNanos = System.nanoTime() / 100L + nanoToMilliOffset.get(); + long offset = currentTime - fromNanos; + if (Math.abs(offset) > MILLIS_TO_UUID * 1000 ) { + nanoToMilliOffset.addAndGet(offset); + } + } + + /** + * Method to overwrite to save the state in a non volatile place. + */ + protected void saveState() { + + } + + /** + * returns a random 47 bit value according to https://tools.ietf.org/html/rfc4122.html#section-4.5 + */ + protected byte[] getNodeIdBytes() { + byte[] idBytes = new byte[6]; + numberGenerator.nextBytes(idBytes); + idBytes[0] |= 0x01; // set multicast bit. + return idBytes; + } + + /** + * Return UUID from UUID.randomUUID(); + */ + @Override + public UUID nextId(Transaction t) { + long current = System.nanoTime() / 100L + nanoToMilliOffset.get(); + + long delta; + int seq; + do { + seq = clockSeq.get(); + while (true) { + long last = timeStamp.get(); + + delta = current - last; + if (delta < -10000 * 20000) { + logger.info("Clock skew of {} ms detected", delta / -10000); + // The clock was adjusted back about 2 seconds, or we were generating a lot of ids too fast + // if so, we try to set the current as last and also increment the clockSeq. + synchronized (this) { + if (clockSeq.compareAndSet(seq, seq + 1)) { + timeStamp.set(current); + saveState(); + computeNanoOffset(); + } else { + continue; + } + } + } + // If current is in the future (most of the cases) try to set it. + if (delta > 0 && timeStamp.compareAndSet(last, current)) { + break; + } else if (timeStamp.compareAndSet(last, last + 1)) { + // here we go, if we pull IDs too fast. + current = last + 1; + break; + } + } + // verify, if this timestamp is for this clock sequence + } while (seq != clockSeq.get()); + + // save state every 60 seconds + if (delta > 60000 * 10000) { + saveState(); + computeNanoOffset(); + } + + long msb = current << 32; // time low + msb |= (current & 0xFFFF00000000L) >> 16; // time mid + msb |= 0x1000 | ((current >> 48) & 0x0FFF); // time hi and version 1 + + byte[] idBytes = getNodeIdBytes(); + + long lsb = 0; + for (int i = 0; i < 6; i++) { + lsb = (lsb << 8) | (idBytes[i] & 0xff); + } + + // RFC 4.1.1. Variant: is set with bits 10xx + seq = seq & 0x3FFF | 0x8000; + lsb |= (long) seq << 48; + + return new UUID(msb, lsb); + } + + /** + * Returns "uuid". + */ + @Override + public String getName() { + return "uuid"; + } + + /** + * Returns false. + */ + @Override + public boolean isDbSequence() { + return false; + } + + /** + * Ignored for UUID as not required as a performance optimisation. + */ + @Override + public void preAllocateIds(int allocateSize) { + // ignored + } + +} diff --git a/src/main/java/io/ebeaninternal/server/idgen/UuidIdGenerator.java b/src/main/java/io/ebeaninternal/server/idgen/UuidV4IdGenerator.java similarity index 68% rename from src/main/java/io/ebeaninternal/server/idgen/UuidIdGenerator.java rename to src/main/java/io/ebeaninternal/server/idgen/UuidV4IdGenerator.java index 06a0f8edc..c7d1d8093 100644 --- a/src/main/java/io/ebeaninternal/server/idgen/UuidIdGenerator.java +++ b/src/main/java/io/ebeaninternal/server/idgen/UuidV4IdGenerator.java @@ -7,10 +7,14 @@ import java.util.UUID; /** * IdGenerator for java util UUID. + * + * This generator generates a 60bit random UUID according to + * https://tools.ietf.org/html/rfc4122.html#section-4.4 + * Use this generator if you want truly random UUIDs */ -public class UuidIdGenerator implements PlatformIdGenerator { +public class UuidV4IdGenerator implements PlatformIdGenerator { - public static final UuidIdGenerator INSTANCE = new UuidIdGenerator(); + public static final UuidV4IdGenerator INSTANCE = new UuidV4IdGenerator(); /** * Return UUID from UUID.randomUUID(); diff --git a/src/test/java/io/ebeaninternal/server/idgen/TestUuidGenerator.java b/src/test/java/io/ebeaninternal/server/idgen/TestUuidGenerator.java new file mode 100644 index 000000000..0f27c5a89 --- /dev/null +++ b/src/test/java/io/ebeaninternal/server/idgen/TestUuidGenerator.java @@ -0,0 +1,141 @@ +/* + * Licensed Materials - Property of FOCONIS AG + * (C) Copyright FOCONIS AG. + */ + +package io.ebeaninternal.server.idgen; + + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.Test; + +import io.ebean.config.dbplatform.PlatformIdGenerator; + + +/** + * Run some simple tests for the UUID generator. + * + * @author Roland Praml, FOCONIS AG + * + */ +public class TestUuidGenerator { + + +/** + * Worker to generate UUIDs + * + * @author Roland Praml, FOCONIS AG + * + */ + public static class IdTest implements Runnable { + private final Map map; + private final PlatformIdGenerator idGen; + private final int ids; + private final AtomicBoolean failFlag; + + public IdTest(PlatformIdGenerator idGen, int ids, Map map, AtomicBoolean failFlag) { + this.idGen = idGen; + this.ids = ids; + this.map = map; + this.failFlag = failFlag; + } + + public IdTest(PlatformIdGenerator idGen, int ids) { + this.idGen = idGen; + this.ids = ids; + this.map = null; + this.failFlag = null; + } + @Override + public void run() { + //System.out.println("Start " + Thread.currentThread()); + for (int i = 0; i < ids; i++) { + UUID uuid = (UUID) idGen.nextId(null); + if (map != null && map.put(uuid, uuid) != null) { + System.err.println("UUID already in set " + uuid); + failFlag.set(true); + } + } + } + } + + /** + * Test takes ~0.3 sec + */ + @Test + public void testUuidV1SingleThread() throws Exception { + testGenerator(1, 1_000_000, UuidV1IdGenerator.getInstance("ebean-test-uuid.state")); + } + + /** + * Test takes ~1.4 sec + */ + @Test + public void testUuidV1MultiThread() throws Exception { + testGenerator(10, 100_000, UuidV1IdGenerator.getInstance("ebean-test-uuid.state")); + } + + /** + * Test takes ~3.1 sec + */ + @Test + public void testUuidV1RndMultiThread() throws Exception { + testGenerator(10, 100_000, UuidV1RndIdGenerator.INSTANCE); + } + + /** + * Test takes ~2.0 sec + */ + @Test + public void testUuidType1RndSingleThread() throws Exception { + testGenerator(1, 1_000_000, UuidV1RndIdGenerator.INSTANCE); + } + + /** + * Test takes ~4.9 sec + */ + @Test + public void testUuidType4MultiThread() throws Exception { + testGenerator(10, 100_000, UuidV4IdGenerator.INSTANCE); + } + + /** + * Test takes ~3.1 sec + */ + @Test + public void testUuidType4SingleThread() throws Exception { + testGenerator(1, 1_000_000, UuidV4IdGenerator.INSTANCE); + } + + + + + private void testGenerator(int threadCount, int count, PlatformIdGenerator generator) throws Exception { + System.out.println("Printing 5 consecutive IDs of " + generator); + for (int i = 0; i < 5; i++) { + System.out.println(generator.nextId(null)); + } + + AtomicBoolean failFlag = new AtomicBoolean(); + ConcurrentHashMap map = new ConcurrentHashMap<>(); + Thread[] threads = new Thread[threadCount]; + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(new IdTest(generator, count, map, failFlag)); + } + for (int i = 0; i < threads.length; i++) { + threads[i].start(); + } + for (int i = 0; i < threads.length; i++) { + threads[i].join(); + } + assertThat(failFlag.get()).isFalse(); + assertThat(map.size()).isEqualTo(threadCount * count); + + } +}