NEW: UUID version 1 generator (#1309)

* NEW: Uuid Type 1 generator

* Remove parameter (from other PR)

* FIX: Mac detection

* Have to use nodeId in generation

src/main/java/io/ebean/config/ServerConfig.java

@@ -372,6 +372,16 @@ public class ServerConfig {
    */
   private DbTypeConfig dbTypeConfig = new DbTypeConfig();
 
+  /**
+   * The UUID version to use.
+   */
+  private UuidVersion uuidVersion = UuidVersion.VERSION4;
+
+  /**
+   * The UUID state file (for Version 1 UUIDs).
+   */
+  private String uuidStateFile = "ebean-uuid.state";
+
   private List<IdGenerator> idGenerators = new ArrayList<>();
   private List<BeanFindController> findControllers = new ArrayList<>();
   private List<BeanPersistController> persistControllers = new ArrayList<>();
@@ -1891,6 +1901,34 @@ public class ServerConfig {
     this.dbTypeConfig.setDbUuid(dbUuid);
   }
 
+  /**
+   * Returns the UUID version mode.
+   */
+  public UuidVersion getUuidVersion() {
+    return uuidVersion;
+  }
+
+  /**
+   * Sets the UUID version mode.
+   */
+  public void setUuidVersion(UuidVersion uuidVersion) {
+    this.uuidVersion = uuidVersion;
+  }
+
+  /**
+   * Return the UUID state file.
+   */
+  public String getUuidStateFile() {
+    return uuidStateFile;
+  }
+
+  /**
+   * Set the UUID state file.
+   */
+  public void setUuidStateFile(String uuidStateFile) {
+    this.uuidStateFile = uuidStateFile;
+  }
+
   /**
    * Return true if LocalTime should be persisted with nanos precision.
    */
@@ -2816,6 +2854,10 @@ public class ServerConfig {
     if (p.getBoolean("uuidStoreAsBinary", false)) {
       dbTypeConfig.setDbUuid(DbUuid.BINARY);
     }
+
+    uuidVersion = p.getEnum(UuidVersion.class, "uuidVersion", uuidVersion);
+    uuidStateFile = p.get("uuidStateFile", uuidStateFile);
+
     localTimeWithNanos = p.getBoolean("localTimeWithNanos", localTimeWithNanos);
     jodaLocalTimeMode = p.get("jodaLocalTimeMode", jodaLocalTimeMode);
 
@@ -3101,4 +3143,10 @@ public class ServerConfig {
       return binaryOptimized;
     }
   }
+
+  public enum UuidVersion {
+    VERSION4,
+    VERSION1,
+    VERSION1RND
+  };
 }

src/main/java/io/ebeaninternal/server/deploy/meta/DeployBeanDescriptor.java

@@ -29,7 +29,9 @@ import io.ebeaninternal.server.deploy.IndexDefinition;
 import io.ebeaninternal.server.deploy.InheritInfo;
 import io.ebeaninternal.server.deploy.TableJoin;
 import io.ebeaninternal.server.deploy.parse.DeployBeanInfo;
-import io.ebeaninternal.server.idgen.UuidIdGenerator;
+import io.ebeaninternal.server.idgen.UuidV1IdGenerator;
+import io.ebeaninternal.server.idgen.UuidV1RndIdGenerator;
+import io.ebeaninternal.server.idgen.UuidV4IdGenerator;
 import io.ebeaninternal.server.rawsql.SpiRawSql;
 
 import javax.persistence.Entity;
@@ -844,8 +846,22 @@ public class DeployBeanDescriptor<T> {
    */
   public void setUuidGenerator() {
     this.idType = IdType.EXTERNAL;
-    this.idGeneratorName = UuidIdGenerator.AUTO_UUID;
-    this.idGenerator = UuidIdGenerator.INSTANCE;
+    this.idGeneratorName = PlatformIdGenerator.AUTO_UUID;
+
+    switch (serverConfig.getUuidVersion()) {
+    case VERSION1:
+      this.idGenerator = UuidV1IdGenerator.getInstance(serverConfig.getUuidStateFile());
+      break;
+
+    case VERSION1RND:
+      this.idGenerator = UuidV1RndIdGenerator.INSTANCE;
+      break;
+
+    case VERSION4:
+    default:
+      this.idGenerator = UuidV4IdGenerator.INSTANCE;
+      break;
+    }
   }
 
   /**

src/main/java/io/ebeaninternal/server/idgen/UuidV1IdGenerator.java

@@ -0,0 +1,197 @@
+package io.ebeaninternal.server.idgen;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.NetworkInterface;
+import java.net.SocketException;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * IdGenerator for java util UUID.
+ *
+ * It extends the UuidV1RndIdGenerator so that it can generate rfc4122 compliant Type 1 UUIDs.
+ *
+ * This generator produces real Type 1 UUIDs (best for sqlserver) - You should use this generator only,
+ * if you can guarantee, that mac addess is uniqe.
+ */
+public class UuidV1IdGenerator extends UuidV1RndIdGenerator {
+
+  private static final Logger logger = LoggerFactory.getLogger("io.ebean.IDGEN");
+
+  private final File stateFile;
+
+  private byte[] nodeId = null;
+
+  private boolean canSaveState = true;
+
+  private static final Map<File, UuidV1IdGenerator> INSTANCES = new ConcurrentHashMap<>();
+
+  /**
+   * Returns an instance for given file.
+   */
+  public static UuidV1IdGenerator getInstance(String file) {
+    return INSTANCES.computeIfAbsent(new File(file), UuidV1IdGenerator::new);
+  }
+
+  /**
+   * Returns an instance for given file.
+   */
+  public static UuidV1IdGenerator getInstance(File file) {
+    return INSTANCES.computeIfAbsent(file, UuidV1IdGenerator::new);
+  }
+
+  /**
+   * Returns an alternative node id - set with the 'ebean.uuid.nodeId' system property.
+   */
+  private static byte[] getAlternativeNodeId() {
+    try {
+      String altNodeId = System.getProperty("ebean.uuid.nodeId");
+      if (altNodeId != null) {
+        String[] components = altNodeId.split("-");
+        if (components.length != 5) {
+          throw new IllegalArgumentException("Invalid nodeId string: " + altNodeId);
+        }
+        byte[] nodeId = new byte[6];
+        for (int i=0; i<5; i++) {
+          nodeId[i] = Byte.decode("0x"+components[i]).byteValue();
+        }
+        return nodeId;
+      }
+    } catch (SecurityException se) {
+      // ignore
+    }
+    return null;
+  }
+
+  /**
+   * Find hardware ID.
+   */
+  private static byte[] getHardwareId() throws SocketException {
+    final Enumeration<NetworkInterface> e = NetworkInterface.getNetworkInterfaces();
+    while (e.hasMoreElements()) {
+      NetworkInterface network = e.nextElement();
+      if (!network.isLoopback()) {
+        return network.getHardwareAddress();
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Creates a new instance of UuidGenerator. Note that there should not be more
+   * than one instance per stateFile.
+   */
+  private UuidV1IdGenerator(final File stateFile) {
+    super();
+    this.stateFile = stateFile;
+    try {
+      // See, if there is an alternative MAC address set.
+      nodeId = getAlternativeNodeId();
+      if (nodeId != null) {
+        logger.info("Using alternative MAC {} to generate Type 1 UUIDs", getNodeIdentifier());
+      } else {
+        nodeId = getHardwareId();
+        logger.info("Using MAC {} to generate Type 1 UUIDs", getNodeIdentifier());
+      }
+      if (nodeId == null) {
+        canSaveState = false;
+        // RFC 4.5 use random portion for node
+        nodeId = super.getNodeIdBytes();
+        logger.error("Have to fall back to random node identifier {} (Reason: No suitable network interface found)", getNodeIdentifier());
+
+      } else {
+        boolean flag = restoreState();
+        UUID uuid = nextId(null);
+        long ts = timeStamp.get();
+        ts -= UUID_EPOCH_OFFSET;
+        ts /= MILLIS_TO_UUID;
+
+        saveState();
+        logger.debug("RestoreState: {}, ClockSeq {}, Timestamp {}, uuid {}, stateFile: {})", flag, clockSeq.get(),
+            new Date(ts), uuid, stateFile);
+      }
+    } catch (IOException e) {
+      canSaveState = false;
+      // RFC 4.5 use random portion for node
+      nodeId = super.getNodeIdBytes();
+      logger.error("Have to fall back to random node identifier {} (Reason: {} )", getNodeIdentifier(), e.getMessage());
+    }
+  }
+
+  /**
+   * Returns the Node-identifier (=MAC address) as string
+   */
+  public String getNodeIdentifier() {
+    if (nodeId == null) {
+      return "none";
+    }
+    StringBuilder sb = new StringBuilder();
+    for (int i = 0; i < nodeId.length; i++) {
+      sb.append(String.format("%02X%s", nodeId[i], (i < nodeId.length - 1) ? "-" : ""));
+    }
+    return sb.toString();
+  }
+
+  /**
+   * Restores the state from the state file.
+   */
+  private boolean restoreState() throws IOException {
+    Properties prop = new Properties();
+    if (stateFile.exists()) {
+      try (InputStream is = new FileInputStream(stateFile)) {
+        prop.load(is);
+      }
+    }
+    if (getNodeIdentifier().equals(prop.getProperty("nodeId"))) {
+      try {
+        Integer seq = Integer.valueOf(prop.getProperty("clockSeq")) & 0x3FFF;
+        Long ts = Long.valueOf(prop.getProperty("timeStamp"));
+        clockSeq.set(seq);
+        timeStamp.set(ts);
+        logger.debug("Restored state from '{}'", stateFile);
+        return true;
+      } catch (NumberFormatException nfe) {
+        // nop
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Saves the state to the state file;
+   */
+  @Override
+  protected void saveState() {
+    if (!canSaveState) {
+      return;
+    }
+    Properties prop = new Properties();
+    prop.setProperty("nodeId", getNodeIdentifier());
+    prop.setProperty("clockSeq", String.valueOf(clockSeq.get()));
+    prop.setProperty("timeStamp", String.valueOf(timeStamp.get()));
+    try (OutputStream os = new FileOutputStream(stateFile)) {
+      prop.store(os, "ebean uuid state file");
+      logger.debug("Persisted state to '{}'", stateFile);
+    } catch (IOException e) {
+      logger.error("Could not persist uuid state to '{}'", stateFile, e);
+    }
+  }
+
+  @Override
+  protected byte[] getNodeIdBytes() {
+    return nodeId;
+  }
+
+}

src/main/java/io/ebeaninternal/server/idgen/UuidV1RndIdGenerator.java

@@ -0,0 +1,179 @@
+package io.ebeaninternal.server.idgen;
+
+import io.ebean.Transaction;
+import io.ebean.config.dbplatform.PlatformIdGenerator;
+
+import java.security.SecureRandom;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * IdGenerator for (pseudo) type 1 UUIDs.
+ *
+ * This implementation generates a type 1 UUID according to
+ * https://tools.ietf.org/html/rfc4122.html#section-4.2
+ * but has no persistence storage. It generates a new random 47 bit node ID with every
+ * UUID.
+ *
+ * Use this, if you want randomness in your UUIDs but want to take advantage of index
+ * optimizations of the database. It may be good with AUTO_BINARY_OPTIMIZED and MySql.
+ *
+ * See: https://www.percona.com/blog/2014/12/19/store-uuid-optimized-way/
+ */
+public class UuidV1RndIdGenerator implements PlatformIdGenerator {
+
+  protected static final Logger logger = LoggerFactory.getLogger("io.ebean.IDGEN");
+
+  // UUID epoch 1582-10-15 00:00:00 and the Unix epoch 1970-01-01 00:00:00.
+  protected static final long UUID_EPOCH_OFFSET = 0x01B21DD213814000L;
+
+  // the resolution is 100ns
+  protected static final long MILLIS_TO_UUID = 10000;
+
+  public static final UuidV1RndIdGenerator INSTANCE = new UuidV1RndIdGenerator();
+
+  protected final AtomicInteger clockSeq = new AtomicInteger((int) (Math.random() * 0x3FFF));
+
+  private final SecureRandom numberGenerator = new SecureRandom();
+
+  protected AtomicLong timeStamp = new AtomicLong(currentUuidTime());
+
+  private AtomicLong nanoToMilliOffset = new AtomicLong(currentUuidTime());
+
+
+  /**
+   * Returns the uuid epoch.
+   *
+   * This is the number of 100ns intervals since 1582-10-15 00:00:00
+   */
+  private static long currentUuidTime() {
+    return (System.currentTimeMillis() * MILLIS_TO_UUID) + UUID_EPOCH_OFFSET;
+  }
+
+  public UuidV1RndIdGenerator() {
+    computeNanoOffset();
+  }
+
+  /**
+   * Computes the internal offset between System.nanoTime() and System.currentTimeMillis().
+   */
+  protected void computeNanoOffset() {
+    long currentTime = currentUuidTime();
+    long fromNanos = System.nanoTime() / 100L + nanoToMilliOffset.get();
+    long offset =  currentTime - fromNanos;
+    if (Math.abs(offset) > MILLIS_TO_UUID * 1000  ) {
+      nanoToMilliOffset.addAndGet(offset);
+    }
+  }
+
+  /**
+   * Method to overwrite to save the state in a non volatile place.
+   */
+  protected void saveState() {
+
+  }
+
+  /**
+   * returns a random 47 bit value according to https://tools.ietf.org/html/rfc4122.html#section-4.5
+   */
+  protected byte[] getNodeIdBytes() {
+    byte[] idBytes = new byte[6];
+    numberGenerator.nextBytes(idBytes);
+    idBytes[0] |= 0x01; // set multicast bit.
+    return idBytes;
+  }
+
+  /**
+   * Return UUID from UUID.randomUUID();
+   */
+  @Override
+  public UUID nextId(Transaction t) {
+    long current = System.nanoTime() / 100L + nanoToMilliOffset.get();
+
+    long delta;
+    int seq;
+    do {
+      seq = clockSeq.get();
+      while (true) {
+        long last = timeStamp.get();
+
+        delta = current - last;
+        if (delta < -10000 * 20000) {
+          logger.info("Clock skew of {} ms detected", delta / -10000);
+          // The clock was adjusted back about 2 seconds, or we were generating a lot of ids too fast
+          // if so, we try to set the current as last and also increment the clockSeq.
+          synchronized (this) {
+            if (clockSeq.compareAndSet(seq, seq + 1)) {
+              timeStamp.set(current);
+              saveState();
+              computeNanoOffset();
+            } else {
+              continue;
+            }
+          }
+        }
+        // If current is in the future (most of the cases) try to set it.
+        if (delta > 0 && timeStamp.compareAndSet(last, current)) {
+          break;
+        } else if (timeStamp.compareAndSet(last, last + 1)) {
+          // here we go, if we pull IDs too fast.
+          current = last + 1;
+          break;
+        }
+      }
+      // verify, if this timestamp is for this clock sequence
+    } while (seq != clockSeq.get());
+
+    // save state every 60 seconds
+    if (delta > 60000 * 10000) {
+      saveState();
+      computeNanoOffset();
+    }
+
+    long msb = current << 32; // time low
+    msb |= (current & 0xFFFF00000000L) >> 16; // time mid
+    msb |= 0x1000 | ((current >> 48) & 0x0FFF); // time hi and version 1
+
+    byte[] idBytes = getNodeIdBytes();
+
+    long lsb = 0;
+    for (int i = 0; i < 6; i++) {
+      lsb = (lsb << 8) | (idBytes[i] & 0xff);
+    }
+
+    // RFC 4.1.1. Variant: is set with bits 10xx
+    seq = seq & 0x3FFF | 0x8000;
+    lsb |= (long) seq << 48;
+
+    return new UUID(msb, lsb);
+  }
+
+  /**
+   * Returns "uuid".
+   */
+  @Override
+  public String getName() {
+    return "uuid";
+  }
+
+  /**
+   * Returns false.
+   */
+  @Override
+  public boolean isDbSequence() {
+    return false;
+  }
+
+  /**
+   * Ignored for UUID as not required as a performance optimisation.
+   */
+  @Override
+  public void preAllocateIds(int allocateSize) {
+    // ignored
+  }
+
+}

src/main/java/io/ebeaninternal/server/idgen/UuidV4IdGenerator.java

@@ -7,10 +7,14 @@ import java.util.UUID;
 
 /**
  * IdGenerator for java util UUID.
+ *
+ * This generator generates a 60bit random UUID according to
+ * https://tools.ietf.org/html/rfc4122.html#section-4.4
+ * Use this generator if you want truly random UUIDs
  */
-public class UuidIdGenerator implements PlatformIdGenerator {
+public class UuidV4IdGenerator implements PlatformIdGenerator {
 
-  public static final UuidIdGenerator INSTANCE = new UuidIdGenerator();
+  public static final UuidV4IdGenerator INSTANCE = new UuidV4IdGenerator();
 
   /**
    * Return UUID from UUID.randomUUID();

src/test/java/io/ebeaninternal/server/idgen/TestUuidGenerator.java

@@ -0,0 +1,141 @@
+/*
+ * Licensed Materials - Property of FOCONIS AG
+ * (C) Copyright FOCONIS AG.
+ */
+
+package io.ebeaninternal.server.idgen;
+
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.junit.Test;
+
+import io.ebean.config.dbplatform.PlatformIdGenerator;
+
+
+/**
+ * Run some simple tests for the UUID generator.
+ *
+ * @author Roland Praml, FOCONIS AG
+ *
+ */
+public class TestUuidGenerator {
+
+
+/**
+ * Worker to generate UUIDs
+ *
+ * @author Roland Praml, FOCONIS AG
+ *
+ */
+  public static class IdTest implements Runnable {
+    private  final Map<UUID, UUID> map;
+    private final PlatformIdGenerator idGen;
+    private final int ids;
+    private final AtomicBoolean failFlag;
+
+    public IdTest(PlatformIdGenerator idGen, int ids, Map<UUID, UUID> map, AtomicBoolean failFlag) {
+      this.idGen = idGen;
+      this.ids = ids;
+      this.map = map;
+      this.failFlag = failFlag;
+    }
+
+    public IdTest(PlatformIdGenerator idGen, int ids) {
+      this.idGen = idGen;
+      this.ids = ids;
+      this.map = null;
+      this.failFlag = null;
+    }
+    @Override
+    public void run() {
+      //System.out.println("Start " + Thread.currentThread());
+      for (int i = 0; i < ids; i++) {
+        UUID uuid = (UUID) idGen.nextId(null);
+        if (map != null && map.put(uuid, uuid) != null) {
+          System.err.println("UUID already in set " + uuid);
+          failFlag.set(true);
+        }
+      }
+    }
+  }
+
+  /**
+   * Test takes ~0.3 sec
+   */
+  @Test
+  public void testUuidV1SingleThread() throws Exception {
+    testGenerator(1, 1_000_000, UuidV1IdGenerator.getInstance("ebean-test-uuid.state"));
+  }
+
+  /**
+   * Test takes ~1.4 sec
+   */
+  @Test
+  public void testUuidV1MultiThread() throws Exception {
+    testGenerator(10, 100_000, UuidV1IdGenerator.getInstance("ebean-test-uuid.state"));
+  }
+
+  /**
+   * Test takes ~3.1 sec
+   */
+  @Test
+  public void testUuidV1RndMultiThread() throws Exception {
+    testGenerator(10, 100_000, UuidV1RndIdGenerator.INSTANCE);
+  }
+
+  /**
+   * Test takes ~2.0 sec
+   */
+  @Test
+  public void testUuidType1RndSingleThread() throws Exception {
+    testGenerator(1, 1_000_000, UuidV1RndIdGenerator.INSTANCE);
+  }
+
+  /**
+   * Test takes ~4.9 sec
+   */
+  @Test
+  public void testUuidType4MultiThread() throws Exception {
+    testGenerator(10, 100_000, UuidV4IdGenerator.INSTANCE);
+  }
+
+  /**
+   * Test takes ~3.1 sec
+   */
+  @Test
+  public void testUuidType4SingleThread() throws Exception {
+    testGenerator(1, 1_000_000, UuidV4IdGenerator.INSTANCE);
+  }
+
+
+
+
+  private void testGenerator(int threadCount, int count, PlatformIdGenerator generator) throws Exception {
+    System.out.println("Printing 5 consecutive IDs of " + generator);
+    for (int i = 0; i < 5; i++) {
+      System.out.println(generator.nextId(null));
+    }
+
+    AtomicBoolean failFlag = new AtomicBoolean();
+    ConcurrentHashMap<UUID, UUID> map = new ConcurrentHashMap<>();
+    Thread[] threads = new Thread[threadCount];
+    for (int i = 0; i < threads.length; i++) {
+      threads[i] = new Thread(new IdTest(generator, count, map, failFlag));
+    }
+    for (int i = 0; i < threads.length; i++) {
+      threads[i].start();
+    }
+    for (int i = 0; i < threads.length; i++) {
+      threads[i].join();
+    }
+    assertThat(failFlag.get()).isFalse();
+    assertThat(map.size()).isEqualTo(threadCount * count);
+
+  }
+}