Merge remote-tracking branch 'ebean/master' into pr/bugfix/connection_closed_on_clob

ebean-core/src/main/java/io/ebeaninternal/api/BindParams.java

@@ -155,7 +155,6 @@ public class BindParams implements Serializable {
    * Set an In Out parameter using position.
    */
   public void setParameter(int position, Object value, int outType) {
-
     Param p = getParam(position);
     p.setInValue(value);
     p.setOutType(outType);
@@ -178,8 +177,8 @@ public class BindParams implements Serializable {
    * Using position set the In value of a parameter. Note that for nulls you
    * must use setNullParameter.
    */
+  @SuppressWarnings("rawtypes")
   public void setParameter(int position, Object value) {
-
     Param p = getParam(position);
     if (value instanceof Collection) {
       // use of postgres ANY with positioned parameter
@@ -214,7 +213,6 @@ public class BindParams implements Serializable {
    * Set a named In Out parameter.
    */
   public void setParameter(String name, Object value, int outType) {
-
     Param p = getParam(name);
     p.setInValue(value);
     p.setOutType(outType);
@@ -232,7 +230,6 @@ public class BindParams implements Serializable {
    * Set a named In parameter that is not null.
    */
   public Param setParameter(String name, Object value) {
-
     Param p = getParam(name);
     p.setInValue(value);
     return p;
@@ -299,7 +296,6 @@ public class BindParams implements Serializable {
    * Return true if the bind hash and count has not changed.
    */
   public boolean isSameBindHash() {
-
     if (bindHash == null) {
       bindHash = calcQueryPlanHash();
       return false;
@@ -330,10 +326,6 @@ public class BindParams implements Serializable {
 
     private final StringBuilder preparedSql;
 
-    public OrderedList() {
-      this(new ArrayList<>());
-    }
-
     public OrderedList(List<Param> paramList) {
       this.paramList = paramList;
       this.preparedSql = new StringBuilder();

ebean-core/src/main/java/io/ebeaninternal/api/LoadManyRequest.java

@@ -71,7 +71,6 @@ public class LoadManyRequest extends LoadRequest {
    * This for use when lazy loading is invoked on methods such as clear() and removeAll() where it
    * generally makes sense to only fetch the Id values as the other property information is not
    * used.
-   * </p>
    */
   private boolean isOnlyIds() {
     return onlyIds;
@@ -91,18 +90,16 @@ public class LoadManyRequest extends LoadRequest {
     return loadContext.getBatchSize();
   }
 
-  private List<Object> getParentIdList() {
-
+  private List<Object> parentIdList(SpiEbeanServer server) {
     List<Object> idList = new ArrayList<>();
-
     BeanPropertyAssocMany<?> many = getMany();
     for (BeanCollection<?> bc : batch) {
       idList.add(many.getParentId(bc.getOwnerBean()));
+      bc.setLoader(server); // don't use the load buffer again
     }
     if (many.getTargetDescriptor().isPadInExpression()) {
       BindPadding.padIds(idList);
     }
-
     return idList;
   }
 
@@ -111,9 +108,7 @@ public class LoadManyRequest extends LoadRequest {
   }
 
   public SpiQuery<?> createQuery(SpiEbeanServer server) {
-
     BeanPropertyAssocMany<?> many = getMany();
-
     SpiQuery<?> query = many.newQuery(server);
     String orderBy = many.getLazyFetchOrderBy();
     if (orderBy != null) {
@@ -124,11 +119,11 @@ public class LoadManyRequest extends LoadRequest {
     if (extraWhere != null) {
       // replace special ${ta} placeholder with the base table alias
       // which is always t0 and add the extra where clause
-      query.where().raw(extraWhere.replace("${ta}", "t0"));
+      query.where().raw(extraWhere.replace("${ta}", "t0").replace("${mta}", "int_"));
     }
 
     query.setLazyLoadForParents(many);
-    many.addWhereParentIdIn(query, getParentIdList(), loadContext.isUseDocStore());
+    many.addWhereParentIdIn(query, parentIdList(server), loadContext.isUseDocStore());
     query.setPersistenceContext(loadContext.getPersistenceContext());
 
     String mode = isLazy() ? "+lazy" : "+query";
@@ -146,7 +141,6 @@ public class LoadManyRequest extends LoadRequest {
       // override to just select the Id values
       query.select(many.getTargetIdProperty());
     }
-
     return query;
   }
 
@@ -154,10 +148,8 @@ public class LoadManyRequest extends LoadRequest {
    * After the query execution check for empty collections and load L2 cache if desired.
    */
   public void postLoad() {
-
     BeanDescriptor<?> desc = loadContext.getBeanDescriptor();
     BeanPropertyAssocMany<?> many = getMany();
-
     // check for BeanCollection's that where never processed
     // in the +query or +lazy load due to no rows (predicates)
     for (BeanCollection<?> bc : batch) {
@@ -172,6 +164,5 @@ public class LoadManyRequest extends LoadRequest {
         desc.cacheManyPropPut(many, bc, parentId);
       }
     }
-
   }
 }

ebean-core/src/main/java/io/ebeaninternal/api/SpiBeanType.java

@@ -0,0 +1,17 @@
+package io.ebeaninternal.api;
+
+import io.ebean.bean.EntityBean;
+
+/**
+ * SPI interface for underlying BeanDescriptor.
+ */
+public interface SpiBeanType {
+
+  /**
+   * Return true if the bean contains a many property that has modifications.
+   * <p>
+   * That is a ManyToMany or a OneToMany with orphan removal with additions
+   * or removals from the collection.
+   */
+  boolean isToManyDirty(EntityBean bean);
+}

ebean-core/src/main/java/io/ebeaninternal/api/SpiBeanTypeManager.java

@@ -0,0 +1,13 @@
+package io.ebeaninternal.api;
+
+/**
+ * Manager of SpiBeanTypes.
+ */
+public interface SpiBeanTypeManager {
+
+  /**
+   * Return the bean type for the given entity class.
+   */
+  SpiBeanType getBeanType(Class<?> entityType);
+
+}

ebean-core/src/main/java/io/ebeaninternal/api/SpiCancelableQuery.java

@@ -0,0 +1,26 @@
+package io.ebeaninternal.api;
+
+import javax.persistence.PersistenceException;
+
+import io.ebean.CancelableQuery;
+
+/**
+ * Cancellable query, that has a delegate.
+ *
+ * @author Roland Praml, FOCONIS AG
+ *
+ */
+public interface SpiCancelableQuery extends CancelableQuery {
+
+  /**
+   * Checks if the query was cancelled.
+   * @throws PersistenceException if query was cancelled.
+   */
+  void checkCancelled();
+
+  /**
+   * Set the underlying cancelable query (with the PreparedStatement).
+   */
+  void setCancelableQuery(CancelableQuery cancelableQuery);
+
+}

ebean-core/src/main/java/io/ebeaninternal/api/SpiEbeanServer.java

@@ -1,14 +1,6 @@
 package io.ebeaninternal.api;
 
-import io.ebean.DtoQuery;
-import io.ebean.EbeanServer;
-import io.ebean.ExtendedServer;
-import io.ebean.PersistenceContextScope;
-import io.ebean.Query;
-import io.ebean.RowConsumer;
-import io.ebean.RowMapper;
-import io.ebean.Transaction;
-import io.ebean.TxScope;
+import io.ebean.*;
 import io.ebean.bean.BeanCollectionLoader;
 import io.ebean.bean.CallOrigin;
 import io.ebean.config.DatabaseConfig;
@@ -25,6 +17,7 @@ import io.ebeaninternal.server.transaction.RemoteTransactionEvent;
 import java.util.List;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
+import java.util.stream.Stream;
 
 /**
  * Service Provider extension to EbeanServer.
@@ -234,6 +227,11 @@ public interface SpiEbeanServer extends ExtendedServer, EbeanServer, BeanCollect
    */
   <T> List<T> findSingleAttributeList(SpiSqlQuery query, Class<T> cls);
 
+  /**
+   * SqlQuery find single attribute streaming the result to a consumer.
+   */
+  <T> void findSingleAttributeEach(SpiSqlQuery query, Class<T> cls, Consumer<T> consumer);
+
   /**
    * SqlQuery find one with mapper.
    */
@@ -249,6 +247,16 @@ public interface SpiEbeanServer extends ExtendedServer, EbeanServer, BeanCollect
    */
   void findEachRow(SpiSqlQuery query, RowConsumer consumer);
 
+  /**
+   * DTO findIterate query.
+   */
+  <T> QueryIterator<T> findDtoIterate(SpiDtoQuery<T> query);
+
+  /**
+   * DTO findStream query.
+   */
+  <T> Stream<T> findDtoStream(SpiDtoQuery<T> query);
+
   /**
    * DTO findList query.
    */

ebean-core/src/main/java/io/ebeaninternal/api/SpiExpression.java

@@ -105,4 +105,9 @@ public interface SpiExpression extends Expression {
    * Check for match to a natural key query returning false if it doesn't match.
    */
   boolean naturalKey(NaturalKeyQueryData<?> data);
+
+  /**
+   * Apply property prefix when filterMany expressions included into main query.
+   */
+  void prefixProperty(String path);
 }

ebean-core/src/main/java/io/ebeaninternal/api/SpiExpressionList.java

@@ -43,4 +43,9 @@ public interface SpiExpressionList<T> extends ExpressionList<T>, SpiExpression {
   default void applyRowLimits(SpiQuery<?> query) {
     // do nothing by default
   }
+
+  /**
+   * Apply property prefix when filterMany expressions included in main query.
+   */
+  void prefixProperty(String path);
 }

ebean-core/src/main/java/io/ebeaninternal/api/SpiPersistenceContext.java

@@ -0,0 +1,17 @@
+package io.ebeaninternal.api;
+
+import io.ebean.bean.PersistenceContext;
+
+import java.util.List;
+
+/**
+ * SPI extension to PersistenceContext.
+ */
+public interface SpiPersistenceContext extends PersistenceContext {
+
+  /**
+   * Return the list of dirty beans held by this persistence context.
+   */
+  List<Object> dirtyBeans(SpiBeanTypeManager manager);
+
+}

ebean-core/src/main/java/io/ebeaninternal/api/SpiQuery.java

@@ -17,7 +17,6 @@ import io.ebeaninternal.server.core.SpiOrmQueryRequest;
 import io.ebeaninternal.server.deploy.BeanDescriptor;
 import io.ebeaninternal.server.deploy.BeanPropertyAssocMany;
 import io.ebeaninternal.server.deploy.TableJoin;
-import io.ebeaninternal.server.query.CancelableQuery;
 import io.ebeaninternal.server.querydefn.NaturalKeyBindParam;
 import io.ebeaninternal.server.querydefn.OrmQueryDetail;
 import io.ebeaninternal.server.querydefn.OrmQueryProperties;
@@ -31,7 +30,7 @@ import java.util.Set;
 /**
  * Object Relational query - Internal extension to Query object.
  */
-public interface SpiQuery<T> extends Query<T>, SpiQueryFetch, TxnProfileEventCodes {
+public interface SpiQuery<T> extends Query<T>, SpiQueryFetch, TxnProfileEventCodes, SpiCancelableQuery {
 
   enum Mode {
     NORMAL(false), LAZYLOAD_MANY(false), LAZYLOAD_BEAN(true), REFRESH_BEAN(true);
@@ -847,16 +846,6 @@ public interface SpiQuery<T> extends Query<T>, SpiQueryFetch, TxnProfileEventCod
    */
   ReadEvent getFutureFetchAudit();
 
-  /**
-   * Set the underlying cancelable query (with the PreparedStatement).
-   */
-  void setCancelableQuery(CancelableQuery cancelableQuery);
-
-  /**
-   * Return true if this query has been cancelled.
-   */
-  boolean isCancelled();
-
   /**
    * Return the base table to use if user defined on the query.
    */

ebean-core/src/main/java/io/ebeaninternal/api/SpiSqlBinding.java

@@ -3,7 +3,7 @@ package io.ebeaninternal.api;
 /**
  * SQL query binding (for SqlQuery and DtoQuery).
  */
-public interface SpiSqlBinding {
+public interface SpiSqlBinding extends SpiCancelableQuery {
 
   /**
    * Return the named or positioned parameters.

ebean-core/src/main/java/io/ebeaninternal/api/SpiTransaction.java

@@ -3,7 +3,6 @@ package io.ebeaninternal.api;
 import io.ebean.ProfileLocation;
 import io.ebean.Transaction;
 import io.ebean.annotation.DocStoreMode;
-import io.ebean.bean.PersistenceContext;
 import io.ebean.event.changelog.BeanChange;
 import io.ebean.event.changelog.ChangeSet;
 import io.ebeaninternal.server.core.PersistDeferredRelationship;
@@ -196,7 +195,7 @@ public interface SpiTransaction extends Transaction {
    * later. This is along the lines of 'extended persistence context'
    * behaviour.
    */
-  PersistenceContext getPersistenceContext();
+  SpiPersistenceContext getPersistenceContext();
 
   /**
    * Set the persistence context to this transaction.
@@ -208,7 +207,7 @@ public interface SpiTransaction extends Transaction {
    * and setPersistenceContext() enable a developer to reuse a single
    * PersistenceContext with multiple transactions.
    */
-  void setPersistenceContext(PersistenceContext context);
+  void setPersistenceContext(SpiPersistenceContext context);
 
   /**
    * Return the underlying Connection for internal use.

ebean-core/src/main/java/io/ebeaninternal/api/SpiTransactionProxy.java

@@ -3,7 +3,6 @@ package io.ebeaninternal.api;
 import io.ebean.ProfileLocation;
 import io.ebean.TransactionCallback;
 import io.ebean.annotation.DocStoreMode;
-import io.ebean.bean.PersistenceContext;
 import io.ebean.event.changelog.BeanChange;
 import io.ebean.event.changelog.ChangeSet;
 import io.ebeaninternal.server.core.PersistDeferredRelationship;
@@ -379,12 +378,12 @@ public abstract class SpiTransactionProxy implements SpiTransaction {
   }
 
   @Override
-  public PersistenceContext getPersistenceContext() {
+  public SpiPersistenceContext getPersistenceContext() {
     return transaction.getPersistenceContext();
   }
 
   @Override
-  public void setPersistenceContext(PersistenceContext context) {
+  public void setPersistenceContext(SpiPersistenceContext context) {
     transaction.setPersistenceContext(context);
   }
 

ebean-core/src/main/java/io/ebeaninternal/server/core/AbstractSqlQueryRequest.java

@@ -1,14 +1,10 @@
 package io.ebeaninternal.server.core;
 
-import io.ebean.EbeanServer;
+import io.ebean.CancelableQuery;
 import io.ebean.Transaction;
 import io.ebean.util.JdbcClose;
-import io.ebeaninternal.api.BindParams;
-import io.ebeaninternal.api.SpiEbeanServer;
-import io.ebeaninternal.api.SpiQuery;
-import io.ebeaninternal.api.SpiSqlBinding;
-import io.ebeaninternal.api.SpiTransaction;
-import io.ebeaninternal.server.lib.Str;
+import io.ebeaninternal.api.*;
+import io.ebeaninternal.server.util.Str;
 import io.ebeaninternal.server.persist.Binder;
 import io.ebeaninternal.server.persist.TrimLogSql;
 import io.ebeaninternal.server.util.BindParamsParser;
@@ -17,11 +13,14 @@ import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.concurrent.locks.ReentrantLock;
+
+import javax.persistence.PersistenceException;
 
 /**
  * Wraps the objects involved in executing a SQL / Relational Query.
  */
-public abstract class AbstractSqlQueryRequest {
+public abstract class AbstractSqlQueryRequest implements CancelableQuery {
 
   protected final SpiSqlBinding query;
 
@@ -41,6 +40,8 @@ public abstract class AbstractSqlQueryRequest {
 
   protected long startNano;
 
+  private final ReentrantLock lock = new ReentrantLock();
+  
   /**
    * Create the BeanFindRequest.
    */
@@ -48,6 +49,7 @@ public abstract class AbstractSqlQueryRequest {
     this.server = server;
     this.query = query;
     this.transaction = (SpiTransaction) t;
+    this.query.setCancelableQuery(this);
   }
 
   /**
@@ -83,11 +85,6 @@ public abstract class AbstractSqlQueryRequest {
     return transaction.isLogSql();
   }
 
-  /**
-   * Set the resultSet and associated query plan if known.
-   */
-  abstract void setResultSet(ResultSet resultSet, Object queryPlanKey) throws SQLException;
-
   /**
    * Return the bindLog for this request.
    */
@@ -95,12 +92,15 @@ public abstract class AbstractSqlQueryRequest {
     return bindLog;
   }
 
+  /**
+   * Set the resultSet and associated query plan if known.
+   */
+  abstract void setResultSet(ResultSet resultSet, Object queryPlanKey) throws SQLException;
+
   /**
    * Return true if we can navigate to the next row.
    */
-  public boolean next() throws SQLException {
-    return resultSet.next();
-  }
+  public abstract boolean next() throws SQLException;
 
   protected abstract void requestComplete();
 
@@ -113,7 +113,6 @@ public abstract class AbstractSqlQueryRequest {
     JdbcClose.close(pstmt);
   }
 
-
   /**
    * Prepare the SQL taking into account named bind parameters.
    */
@@ -145,24 +144,30 @@ public abstract class AbstractSqlQueryRequest {
   }
 
   protected void executeAsSql(Binder binder) throws SQLException {
-    prepareSql();
-    Connection conn = transaction.getInternalConnection();
-    pstmt = conn.prepareStatement(sql);
-    if (query.getTimeout() > 0) {
-      pstmt.setQueryTimeout(query.getTimeout());
+    lock.lock();
+    try {
+      query.checkCancelled();
+      prepareSql();
+      Connection conn = transaction.getInternalConnection();
+      pstmt = conn.prepareStatement(sql);
+      if (query.getTimeout() > 0) {
+        pstmt.setQueryTimeout(query.getTimeout());
+      }
+      if (query.getBufferFetchSizeHint() > 0) {
+        pstmt.setFetchSize(query.getBufferFetchSizeHint());
+      }
+      BindParams bindParams = query.getBindParams();
+      if (!bindParams.isEmpty()) {
+        this.bindLog = binder.bind(bindParams, pstmt, conn);
+      }
+      if (isLogSql()) {
+        transaction.logSql(Str.add(TrimLogSql.trim(sql), "; --bind(", bindLog, ")"));
+      }
+    } finally {
+      lock.unlock();
     }
-    if (query.getBufferFetchSizeHint() > 0) {
-      pstmt.setFetchSize(query.getBufferFetchSizeHint());
-    }
-    BindParams bindParams = query.getBindParams();
-    if (!bindParams.isEmpty()) {
-      this.bindLog = binder.bind(bindParams, pstmt, conn);
-    }
-    if (isLogSql()) {
-      transaction.logSql(Str.add(TrimLogSql.trim(sql), "; --bind(", bindLog, ")"));
-    }
-
     setResultSet(pstmt.executeQuery(), null);
+    query.checkCancelled();
   }
 
   /**
@@ -172,4 +177,13 @@ public abstract class AbstractSqlQueryRequest {
     return sql;
   }
 
+  @Override
+  public void cancel() {
+    lock.lock();
+    try {
+      JdbcClose.cancel(pstmt);
+    } finally {
+      lock.unlock();
+    }
+  }
 }

ebean-core/src/main/java/io/ebeaninternal/server/core/DefaultServer.java

@@ -1382,7 +1382,7 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
   @Nonnull
   @Override
   public <T> FutureList<T> findFutureList(Query<T> query, Transaction t) {
-    SpiQuery<T> spiQuery = (SpiQuery<T>) query;
+    SpiQuery<T> spiQuery = (SpiQuery<T>) query.copy();
     spiQuery.setFutureFetch(true);
     // FutureList query always run in it's own persistence content
     spiQuery.setPersistenceContext(new DefaultPersistenceContext());
@@ -1433,32 +1433,11 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
   @Nonnull
   @Override
   public <T> Stream<T> findStream(Query<T> query, Transaction transaction) {
-    SpiOrmQueryRequest<T> request = createQueryRequest(Type.ITERATE, query, transaction);
-    try {
-      request.initTransIfRequired();
-      return toStream(request.findIterate());
-    } catch (RuntimeException ex) {
-      request.endTransIfRequired();
-      throw ex;
-    }
+    return toStream(findIterate(query, transaction));
   }
 
   private <T> Stream<T> toStream(QueryIterator<T> queryIterator) {
-    return stream(spliteratorUnknownSize(queryIterator, Spliterator.ORDERED), false)
-      .onClose(new QueryIteratorClose(queryIterator));
-  }
-
-  private static class QueryIteratorClose implements Runnable {
-    private final QueryIterator<?> iterator;
-
-    private QueryIteratorClose(QueryIterator<?> iterator) {
-      this.iterator = iterator;
-    }
-
-    @Override
-    public void run() {
-      iterator.close();
-    }
+    return stream(spliteratorUnknownSize(queryIterator, Spliterator.ORDERED), false).onClose(queryIterator::close);
   }
 
   @Override
@@ -1606,6 +1585,11 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
     return executeSqlQuery((req) -> req.findOneMapper(mapper), query);
   }
 
+  @Override
+  public <T> void findSingleAttributeEach(SpiSqlQuery query, Class<T> cls, Consumer<T> consumer) {
+    executeSqlQuery((req) -> req.findSingleAttributeEach(cls, consumer), query);
+  }
+
   @Override
   public <T> List<T> findSingleAttributeList(SpiSqlQuery query, Class<T> cls) {
     return executeSqlQuery((req) -> req.findSingleAttributeList(cls), query);
@@ -1649,6 +1633,23 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
     }
   }
 
+  @Override
+  public <T> QueryIterator<T> findDtoIterate(SpiDtoQuery<T> query) {
+    DtoQueryRequest<T> request = new DtoQueryRequest<>(this, dtoQueryEngine, query);
+    try {
+      request.initTransIfRequired();
+      return request.findIterate();
+    } catch (RuntimeException ex) {
+      request.endTransIfRequired();
+      throw ex;
+    }
+  }
+
+  @Override
+  public <T> Stream<T> findDtoStream(SpiDtoQuery<T> query) {
+    return toStream(findDtoIterate(query));
+  }
+
   @Override
   public <T> List<T> findDtoList(SpiDtoQuery<T> query) {
     DtoQueryRequest<T> request = new DtoQueryRequest<>(this, dtoQueryEngine, query);

ebean-core/src/main/java/io/ebeaninternal/server/core/DtoQueryRequest.java

@@ -1,5 +1,6 @@
 package io.ebeaninternal.server.core;
 
+import io.ebean.QueryIterator;
 import io.ebean.core.type.DataReader;
 import io.ebeaninternal.api.SpiDtoQuery;
 import io.ebeaninternal.api.SpiEbeanServer;
@@ -52,6 +53,7 @@ public final class DtoQueryRequest<T> extends AbstractSqlQueryRequest {
       ormQuery.setType(type);
       ormQuery.setManualId();
 
+      query.setCancelableQuery(ormQuery);
       // execute the underlying ORM query returning the ResultSet
       SpiResultSet result = server.findResultSet(ormQuery, transaction);
       this.pstmt = result.getStatement();
@@ -90,6 +92,11 @@ public final class DtoQueryRequest<T> extends AbstractSqlQueryRequest {
     }
   }
 
+  public QueryIterator<T> findIterate() {
+    flushJdbcBatchOnQuery();
+    return queryEngine.findIterate(this);
+  }
+
   public void findEach(Consumer<T> consumer) {
     flushJdbcBatchOnQuery();
     queryEngine.findEach(this, consumer);
@@ -110,9 +117,13 @@ public final class DtoQueryRequest<T> extends AbstractSqlQueryRequest {
     return queryEngine.findList(this);
   }
 
+  public boolean next() throws SQLException {
+    query.checkCancelled();
+    return dataReader.next();
+  }
+
   @SuppressWarnings("unchecked")
   public T readNextBean() throws SQLException {
-    dataReader.resetColumnPosition();
     return (T) plan.readRow(dataReader);
   }
 

ebean-core/src/main/java/io/ebeaninternal/server/core/OrmQueryRequest.java

@@ -1,6 +1,7 @@
 package io.ebeaninternal.server.core;
 
 import io.ebean.CacheMode;
+import io.ebean.CancelableQuery;
 import io.ebean.OrderBy;
 import io.ebean.PersistenceContextScope;
 import io.ebean.QueryIterator;
@@ -35,7 +36,6 @@ import io.ebeaninternal.server.deploy.DeployPropertyParserMap;
 import io.ebeaninternal.server.el.ElPropertyValue;
 import io.ebeaninternal.server.loadcontext.DLoadContext;
 import io.ebeaninternal.server.query.CQueryPlan;
-import io.ebeaninternal.server.query.CancelableQuery;
 import io.ebeaninternal.server.transaction.DefaultPersistenceContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -219,10 +219,10 @@ public final class OrmQueryRequest<T> extends BeanRequest implements SpiOrmQuery
    */
   @Override
   public void prepareQuery() {
+    secondaryQueries = query.convertJoins();
     beanDescriptor.prepareQuery(query);
     adapterPreQuery();
-    this.secondaryQueries = query.convertJoins();
-    this.queryPlanKey = query.prepare(this);
+    queryPlanKey = query.prepare(this);
   }
 
   public boolean isNativeSql() {

ebean-core/src/main/java/io/ebeaninternal/server/core/PersistRequestBean.java

@@ -379,9 +379,11 @@ public final class PersistRequestBean<T> extends PersistRequest implements BeanP
   public void setBatched() {
     batched = true;
     if (type == Type.INSERT || type == Type.UPDATE) {
-      // used to trigger automatic jdbc batch flush
-      intercept.registerGetterCallback(this);
-      getterCallback = true;
+      if (beanDescriptor.hasSingleIdProperty()) {
+        // used to trigger automatic jdbc batch flush
+        intercept.registerGetterCallback(this);
+        getterCallback = true;
+      }
     }
   }
 

ebean-core/src/main/java/io/ebeaninternal/server/core/PersistRequestUpdateSql.java

@@ -3,7 +3,7 @@ package io.ebeaninternal.server.core;
 import io.ebeaninternal.api.SpiEbeanServer;
 import io.ebeaninternal.api.SpiSqlUpdate;
 import io.ebeaninternal.api.SpiTransaction;
-import io.ebeaninternal.server.lib.Str;
+import io.ebeaninternal.server.util.Str;
 import io.ebeaninternal.server.persist.BatchControl;
 import io.ebeaninternal.server.persist.PersistExecute;
 import io.ebeaninternal.server.persist.TrimLogSql;

ebean-core/src/main/java/io/ebeaninternal/server/core/RelationalQueryEngine.java

@@ -20,17 +20,22 @@ public interface RelationalQueryEngine {
   /**
    * Find a list of beans using relational query.
    */
-  List<SqlRow> findList(RelationalQueryRequest request);
-
-  /**
-   * Find each query using relational query.
-   */
-  void findEach(RelationalQueryRequest request, Consumer<SqlRow> consumer);
+  <T> List<T> findList(RelationalQueryRequest request, RowReader<T> reader);
 
   /**
    * Find each while query using relational query.
    */
-  void findEach(RelationalQueryRequest request, Predicate<SqlRow> consumer);
+  <T> void findEach(RelationalQueryRequest request, RowReader<T> reader, Predicate<T> consumer);
+
+  /**
+   * Find each via raw consumer.
+   */
+  void findEach(RelationalQueryRequest request, RowConsumer mapper);
+
+  /**
+   * Find one via mapper.
+   */
+  <T> T findOne(RelationalQueryRequest request, RowMapper<T> mapper);
 
   /**
    * Find single attribute.
@@ -43,19 +48,9 @@ public interface RelationalQueryEngine {
   <T> List<T> findSingleAttributeList(RelationalQueryRequest request, Class<T> cls);
 
   /**
-   * Find one via mapper.
+   * Find single attribute streaming the result to a consumer.
    */
-  <T> T findOneMapper(RelationalQueryRequest request, RowMapper<T> mapper);
-
-  /**
-   * Find list via mapper.
-   */
-  <T> List<T> findListMapper(RelationalQueryRequest request, RowMapper<T> mapper);
-
-  /**
-   * Find each via raw consumer.
-   */
-  void findEachRow(RelationalQueryRequest request, RowConsumer mapper);
+  <T> void findSingleAttributeEach(RelationalQueryRequest request, Class<T> cls, Consumer<T> consumer);
 
   /**
    * Collect SQL query execution statistics.

ebean-core/src/main/java/io/ebeaninternal/server/core/RelationalQueryRequest.java

@@ -1,10 +1,6 @@
 package io.ebeaninternal.server.core;
 
-import io.ebean.RowConsumer;
-import io.ebean.RowMapper;
-import io.ebean.SqlQuery;
-import io.ebean.SqlRow;
-import io.ebean.Transaction;
+import io.ebean.*;
 import io.ebeaninternal.api.SpiEbeanServer;
 import io.ebeaninternal.api.SpiSqlBinding;
 
@@ -57,18 +53,24 @@ public final class RelationalQueryRequest extends AbstractSqlQueryRequest {
 
   boolean findEachRow(RowConsumer mapper) {
     flushJdbcBatchOnQuery();
-    queryEngine.findEachRow(this, mapper);
+    queryEngine.findEach(this, mapper);
     return true;
   }
 
   <T> List<T> findListMapper(RowMapper<T> mapper) {
     flushJdbcBatchOnQuery();
-    return queryEngine.findListMapper(this, mapper);
+    return queryEngine.findList(this, () -> mapper.map(resultSet, rows++));
   }
 
   <T> T findOneMapper(RowMapper<T> mapper) {
     flushJdbcBatchOnQuery();
-    return queryEngine.findOneMapper(this, mapper);
+    return queryEngine.findOne(this, mapper);
+  }
+
+  public <T> boolean findSingleAttributeEach(Class<T> cls, Consumer<T> consumer) {
+    flushJdbcBatchOnQuery();
+    queryEngine.findSingleAttributeEach(this, cls, consumer);
+    return true;
   }
 
   public <T> List<T> findSingleAttributeList(Class<T> cls) {
@@ -83,17 +85,17 @@ public final class RelationalQueryRequest extends AbstractSqlQueryRequest {
 
   public void findEach(Consumer<SqlRow> consumer) {
     flushJdbcBatchOnQuery();
-    queryEngine.findEach(this, consumer);
+    queryEngine.findEach(this, (resultSet, rowNum) -> consumer.accept(createNewRow()));
   }
 
   public void findEachWhile(Predicate<SqlRow> consumer) {
     flushJdbcBatchOnQuery();
-    queryEngine.findEach(this, consumer);
+    queryEngine.findEach(this, this::createNewRow, consumer);
   }
 
   public List<SqlRow> findList() {
     flushJdbcBatchOnQuery();
-    return queryEngine.findList(this);
+    return queryEngine.findList(this, this::createNewRow);
   }
 
   /**
@@ -135,16 +137,15 @@ public final class RelationalQueryRequest extends AbstractSqlQueryRequest {
     return resultSet;
   }
 
-  public void incrementRows() {
-    rows++;
-  }
-
-  public <T> List<T> mapList(RowMapper<T> mapper) throws SQLException {
-    List<T> list = new ArrayList<>();
-    while (next()) {
-      list.add(mapper.map(resultSet, rows++));
+  @Override
+  public boolean next() throws SQLException {
+    query.checkCancelled();
+    if (!resultSet.next()) {
+      return false;
+    } else {
+      rows++;
+      return true;
     }
-    return list;
   }
 
   public <T> T mapOne(RowMapper<T> mapper) throws SQLException {

ebean-core/src/main/java/io/ebeaninternal/server/core/RowReader.java

@@ -0,0 +1,14 @@
+package io.ebeaninternal.server.core;
+
+import java.sql.SQLException;
+
+/**
+ * Read a row building a result for that row.
+ */
+public interface RowReader<T> {
+
+  /**
+   * Build and return a result for a row.
+   */
+  T read() throws SQLException;
+}

ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanDescriptor.java

@@ -38,15 +38,7 @@ import io.ebean.plugin.BeanType;
 import io.ebean.plugin.ExpressionPath;
 import io.ebean.plugin.Property;
 import io.ebean.util.SplitName;
-import io.ebeaninternal.api.BeanCacheResult;
-import io.ebeaninternal.api.CQueryPlanKey;
-import io.ebeaninternal.api.ConcurrencyMode;
-import io.ebeaninternal.api.LoadBeanContext;
-import io.ebeaninternal.api.LoadContext;
-import io.ebeaninternal.api.SpiEbeanServer;
-import io.ebeaninternal.api.SpiQuery;
-import io.ebeaninternal.api.SpiTransaction;
-import io.ebeaninternal.api.SpiUpdatePlan;
+import io.ebeaninternal.api.*;
 import io.ebeaninternal.api.TransactionEventTable.TableIUD;
 import io.ebeaninternal.api.json.SpiJsonReader;
 import io.ebeaninternal.api.json.SpiJsonWriter;
@@ -119,7 +111,7 @@ import static io.ebeaninternal.server.persist.DmlUtil.isNullOrZero;
 /**
  * Describes Beans including their deployment information.
  */
-public class BeanDescriptor<T> implements BeanType<T>, STreeType {
+public class BeanDescriptor<T> implements BeanType<T>, STreeType, SpiBeanType {
 
   private static final Logger logger = LoggerFactory.getLogger(BeanDescriptor.class);
 
@@ -1755,6 +1747,13 @@ public class BeanDescriptor<T> implements BeanType<T>, STreeType {
     return idProperty != null;
   }
 
+  /**
+   * Return false for IdClass case with multiple @Id properties.
+   */
+  public boolean hasSingleIdProperty() {
+    return idPropertyIndex != -1;
+  }
+
   /**
    * Return true if this type has a simple Id and the platform supports mutli-value binding.
    */
@@ -1880,7 +1879,7 @@ public class BeanDescriptor<T> implements BeanType<T>, STreeType {
     if (refBean == null) {
       refBean = createReference(readOnly, false, id, pc);
     }
-    return (EntityBean)refBean;
+    return (EntityBean) refBean;
   }
 
   /**
@@ -2017,6 +2016,13 @@ public class BeanDescriptor<T> implements BeanType<T>, STreeType {
     return owner.getBeanDescriptor(otherType);
   }
 
+  /**
+   * Returns true, if the table is managed (i.e. an existing m2m relation).
+   */
+  public boolean isTableManaged(String tableName) {
+    return owner.isTableManaged(tableName);
+  }
+  
   /**
    * Return the order column property.
    */
@@ -2937,6 +2943,20 @@ public class BeanDescriptor<T> implements BeanType<T>, STreeType {
     return false;
   }
 
+  @Override
+  public boolean isToManyDirty(EntityBean bean) {
+    final EntityBeanIntercept ebi = bean._ebean_getIntercept();
+    for (BeanPropertyAssocMany<?> many : propertiesManySave) {
+      if (ebi.isLoadedProperty(many.getPropertyIndex())) {
+        final BeanCollection<?> value = (BeanCollection<?>) many.getValue(bean);
+        if (value != null && value.hasModifications()) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
   /**
    * Return true if the bean is draftable and considered a 'live' instance.
    */
@@ -3139,7 +3159,8 @@ public class BeanDescriptor<T> implements BeanType<T>, STreeType {
   }
 
   public boolean isIdLoaded(EntityBeanIntercept ebi) {
-    return ebi.isLoadedProperty(idPropertyIndex);
+    // assume id loaded for IdClass case with idPropertyIndex == -1
+    return idPropertyIndex == -1 ? true : ebi.isLoadedProperty(idPropertyIndex);
   }
 
   boolean hasIdValue(EntityBean bean) {
@@ -3179,7 +3200,7 @@ public class BeanDescriptor<T> implements BeanType<T>, STreeType {
       int propertyIndex = beanProperty.getPropertyIndex();
       if (!ebi.isDirtyProperty(propertyIndex) && ebi.isLoadedProperty(propertyIndex)) {
         Object value = beanProperty.getValue(ebi.getOwner());
-        if (value == null || beanProperty.isDirtyValue(value)) {
+        if (value != null && beanProperty.isDirtyValue(value)) {
           // mutable scalar value which is considered dirty so mark
           // it as such so that it is included in an update
           ebi.markPropertyAsChanged(propertyIndex);

ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanDescriptorManager.java

@@ -26,9 +26,7 @@ import io.ebean.meta.MetricVisitor;
 import io.ebean.meta.QueryPlanInit;
 import io.ebean.plugin.BeanType;
 import io.ebean.util.AnnotationUtil;
-import io.ebeaninternal.api.ConcurrencyMode;
-import io.ebeaninternal.api.SpiEbeanServer;
-import io.ebeaninternal.api.TransactionEventTable;
+import io.ebeaninternal.api.*;
 import io.ebeaninternal.server.cache.CacheChangeSet;
 import io.ebeaninternal.server.cache.SpiCacheManager;
 import io.ebeaninternal.server.core.InternString;
@@ -88,7 +86,7 @@ import java.util.concurrent.TimeUnit;
 /**
  * Creates BeanDescriptors.
  */
-public class BeanDescriptorManager implements BeanDescriptorMap {
+public class BeanDescriptorManager implements BeanDescriptorMap, SpiBeanTypeManager {
 
   private static final Logger logger = LoggerFactory.getLogger(BeanDescriptorManager.class);
 
@@ -263,6 +261,11 @@ public class BeanDescriptorManager implements BeanDescriptorMap {
     return descQueueMap.get(queueId);
   }
 
+  @Override
+  public SpiBeanType getBeanType(Class<?> entityType) {
+    return getBeanDescriptor(entityType);
+  }
+
   @Override
   @SuppressWarnings("unchecked")
   public <T> BeanDescriptor<T> getBeanDescriptor(Class<T> entityType) {
@@ -442,6 +445,12 @@ public class BeanDescriptorManager implements BeanDescriptorMap {
     return tableToDescMap.get(tableName.toLowerCase());
   }
 
+  @Override
+  public boolean isTableManaged(String tableName) {
+    return tableToDescMap.get(tableName.toLowerCase()) != null
+        || tableToViewDescMap.get(tableName.toLowerCase()) != null;
+  }
+
   /**
    * Invalidate entity beans based on views via their dependent tables.
    */

ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanDescriptorMap.java

@@ -76,4 +76,10 @@ public interface BeanDescriptorMap {
    * Return true if Jackson core is present on the classpath.
    */
   boolean isJacksonCorePresent();
+  
+  /**
+   * Returns true, if the given table (or view) is managed by ebean
+   * (= an entity exists)
+   */
+  boolean isTableManaged(String tableName);
 }

ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanPropertyAssocMany.java

@@ -439,6 +439,7 @@ public class BeanPropertyAssocMany<T> extends BeanPropertyAssoc<T> implements ST
   @Override
   public String getAssocIsEmpty(SpiExpressionRequest request, String path) {
     boolean softDelete = targetDescriptor.isSoftDelete();
+    boolean needsX2Table = softDelete || getExtraWhere() != null;
     StringBuilder sb = new StringBuilder(50);
     SpiQuery<?> query = request.getQueryRequest().getQuery();
     if (hasJoinTable()) {
@@ -446,7 +447,7 @@ public class BeanPropertyAssocMany<T> extends BeanPropertyAssoc<T> implements ST
     } else {
       sb.append(targetDescriptor.getBaseTable(query.getTemporalMode()));
     }
-    if (softDelete && hasJoinTable()) {
+    if (needsX2Table && hasJoinTable()) {
       sb.append(" x join ");
       sb.append(targetDescriptor.getBaseTable(query.getTemporalMode()));
       sb.append(" x2 on ");
@@ -461,6 +462,16 @@ public class BeanPropertyAssocMany<T> extends BeanPropertyAssoc<T> implements ST
       }
       exportedProperties[i].appendWhere(sb, "x.", path);
     }
+    if (getExtraWhere() != null) {
+      sb.append(" and ");
+      if (hasJoinTable()) {
+        sb.append(getExtraWhere().replace("${ta}", "x2").replace("${mta}", "x"));
+      } else {
+        sb.append(getExtraWhere().replace("${ta}", "x"));
+      }
+    }
+
+    
     if (softDelete) {
       String alias = hasJoinTable() ? "x2" : "x";
       sb.append(" and ").append(targetDescriptor.getSoftDeletePredicate(alias));
@@ -1061,4 +1072,16 @@ public class BeanPropertyAssocMany<T> extends BeanPropertyAssoc<T> implements ST
   public void bindElementValue(SqlUpdate insert, Object value) {
     targetDescriptor.bindElementValue(insert, value);
   }
+  
+  /**
+   * Returns true, if we must create a m2m join table.
+   */
+  public boolean createJoinTable() {
+    if (hasJoinTable() && getMappedBy() == null) {
+      // only create on other 'owning' side
+      return !descriptor.isTableManaged(intersectionJoin.getTable());
+    } else {
+      return false;
+    }
+  }
 }

ebean-core/src/main/java/io/ebeaninternal/server/deploy/DbSqlContext.java

@@ -10,7 +10,7 @@ public interface DbSqlContext {
   /**
    * Add a join to the sql query.
    */
-  void addJoin(String type, String table, TableJoinColumn[] cols, String a1, String a2);
+  void addJoin(String type, String table, TableJoinColumn[] cols, String a1, String a2, String extraWhere);
 
   /**
    * Push the current table alias onto the stack.

ebean-core/src/main/java/io/ebeaninternal/server/deploy/TableJoin.java

@@ -34,6 +34,8 @@ public final class TableJoin {
   private final int queryHash;
 
   private final PropertyForeignKey foreignKey;
+  
+  private final String extraWhere;
 
   public TableJoin(DeployTableJoin deploy) {
     this(deploy, null);
@@ -44,6 +46,7 @@ public final class TableJoin {
    */
   public TableJoin(DeployTableJoin deploy, PropertyForeignKey foreignKey) {
     this.foreignKey = foreignKey;
+    this.extraWhere = deploy.getExtraWhere();
     this.table = InternString.intern(deploy.getTable());
     this.type = deploy.getType();
     this.inheritInfo = deploy.getInheritInfo();
@@ -57,6 +60,7 @@ public final class TableJoin {
 
   private TableJoin(TableJoin source, String overrideColumn) {
     this.foreignKey = null;
+    this.extraWhere = source.extraWhere;
     this.table = source.table;
     this.type = source.type;
     this.inheritInfo = source.inheritInfo;
@@ -146,7 +150,7 @@ public final class TableJoin {
 
   public SqlJoinType addJoin(SqlJoinType joinType, String a1, String a2, DbSqlContext ctx) {
     String joinLiteral = joinType.getLiteral(type);
-    ctx.addJoin(joinLiteral, table, columns(), a1, a2);
+    ctx.addJoin(joinLiteral, table, columns(), a1, a2, extraWhere);
     return joinType.autoToOuter(type);
   }
 

ebean-core/src/main/java/io/ebeaninternal/server/deploy/meta/DeployBeanPropertyAssoc.java

@@ -86,6 +86,7 @@ public abstract class DeployBeanPropertyAssoc<T> extends DeployBeanProperty {
    * collection.
    */
   public void setExtraWhere(String extraWhere) {
+    this.tableJoin.setExtraWhere(extraWhere);
     this.extraWhere = extraWhere;
   }
 

ebean-core/src/main/java/io/ebeaninternal/server/deploy/meta/DeployTableJoin.java

@@ -33,6 +33,8 @@ public class DeployTableJoin {
   private ArrayList<DeployTableJoinColumn> columns = new ArrayList<>(4);
 
   private InheritInfo inheritInfo;
+  
+  private String extraWhere;
 
   /**
    * Create a DeployTableJoin.
@@ -137,6 +139,18 @@ public class DeployTableJoin {
     this.type = type;
   }
 
+  /**
+   * Returns the clause of an extra &#64;Where annotation.
+   * @return
+   */
+  public String getExtraWhere() {
+    return extraWhere;
+  }
+  
+  public void setExtraWhere(String extraWhere) {
+    this.extraWhere = extraWhere;
+  }
+  
   public DeployTableJoin createInverse(String tableName) {
 
     DeployTableJoin inverse = new DeployTableJoin();

ebean-core/src/main/java/io/ebeaninternal/server/deploy/parse/AnnotationAssocManys.java

@@ -118,7 +118,7 @@ class AnnotationAssocManys extends AnnotationAssoc {
 
     Where where = prop.getMetaAnnotationWhere(platform);
     if (where != null) {
-      prop.setExtraWhere(where.clause());
+      prop.setExtraWhere(processFormula(where.clause()));
     }
 
     FetchPreference fetchPreference = get(prop, FetchPreference.class);

ebean-core/src/main/java/io/ebeaninternal/server/deploy/parse/AnnotationAssocOnes.java

@@ -54,7 +54,6 @@ public class AnnotationAssocOnes extends AnnotationAssoc {
   }
 
   private void readAssocOne(DeployBeanPropertyAssocOne<?> prop) {
-
     ManyToOne manyToOne = get(prop, ManyToOne.class);
     if (manyToOne != null) {
       readManyToOne(manyToOne, prop);
@@ -97,7 +96,7 @@ public class AnnotationAssocOnes extends AnnotationAssoc {
     Where where = prop.getMetaAnnotationWhere(platform);
     if (where != null) {
       // not expecting this to be used on assoc one properties
-      prop.setExtraWhere(where.clause());
+      prop.setExtraWhere(processFormula(where.clause()));
     }
 
     PrimaryKeyJoinColumn primaryKeyJoin = get(prop, PrimaryKeyJoinColumn.class);
@@ -183,7 +182,6 @@ public class AnnotationAssocOnes extends AnnotationAssoc {
   }
 
   private void readManyToOne(ManyToOne propAnn, DeployBeanPropertyAssocOne<?> beanProp) {
-
     setCascadeTypes(propAnn.cascade(), beanProp.getCascadeInfo());
     setTargetType(propAnn.targetEntity(), beanProp);
     setBeanTable(beanProp);
@@ -194,7 +192,6 @@ public class AnnotationAssocOnes extends AnnotationAssoc {
   }
 
   private void readOneToOne(OneToOne propAnn, DeployBeanPropertyAssocOne<?> prop) {
-
     prop.setOneToOne();
     prop.setDbInsertable(true);
     prop.setDbUpdateable(true);
@@ -223,21 +220,19 @@ public class AnnotationAssocOnes extends AnnotationAssoc {
   }
 
   private void readPrimaryKeyJoin(PrimaryKeyJoinColumn primaryKeyJoin, DeployBeanPropertyAssocOne<?> prop) {
-
     if (!prop.isOneToOne()) {
       throw new IllegalStateException("Expecting property " + prop.getFullBeanName() + " with PrimaryKeyJoinColumn to be a OneToOne?");
     }
     prop.setPrimaryKeyJoin(true);
 
     if (!primaryKeyJoin.name().isEmpty()) {
-      log.warn("Automatically determining join columns and ignoring PrimaryKeyJoinColumn.name {} on {}", primaryKeyJoin.name(), prop.getFullBeanName());
+      log.info("Automatically determining join columns for @PrimaryKeyJoinColumn - ignoring PrimaryKeyJoinColumn.name attribute [{}] on {}", primaryKeyJoin.name(), prop.getFullBeanName());
     }
     if (!primaryKeyJoin.referencedColumnName().isEmpty()) {
-      log.warn("Automatically determining join columns and Ignoring PrimaryKeyJoinColumn.referencedColumnName {} on {}", primaryKeyJoin.referencedColumnName(), prop.getFullBeanName());
+      log.info("Automatically determining join columns for @PrimaryKeyJoinColumn - Ignoring PrimaryKeyJoinColumn.referencedColumnName attribute [{}] on {}", primaryKeyJoin.referencedColumnName(), prop.getFullBeanName());
     }
 
     BeanTable baseBeanTable = factory.getBeanTable(info.getDescriptor().getBeanType());
-
     String localPrimaryKey = baseBeanTable.getIdColumn();
     String foreignColumn = getBeanTable(prop).getIdColumn();
 
@@ -245,7 +240,6 @@ public class AnnotationAssocOnes extends AnnotationAssoc {
   }
 
   private void readEmbedded(DeployBeanPropertyAssocOne<?> prop, Embedded embedded) {
-
     if (descriptor.isDocStoreOnly() && prop.getDocStoreDoc() == null) {
       prop.setDocStoreEmbedded("");
     }
@@ -257,7 +251,6 @@ public class AnnotationAssocOnes extends AnnotationAssoc {
     } catch (NoSuchMethodError e) {
       // using standard JPA API without prefix option, maybe in EE container
     }
-
     readEmbeddedAttributeOverrides(prop);
   }
 

ebean-core/src/main/java/io/ebeaninternal/server/deploy/parse/AnnotationFields.java

@@ -153,7 +153,7 @@ public class AnnotationFields extends AnnotationParser {
 
     Formula formula = prop.getMetaAnnotationFormula(platform);
     if (formula != null) {
-      prop.setSqlFormula(formula.select(), formula.join());
+      prop.setSqlFormula(processFormula(formula.select()), processFormula(formula.join()));
     }
 
     initWhoProperties(prop);
@@ -334,7 +334,7 @@ public class AnnotationFields extends AnnotationParser {
     }
     Formula formula = prop.getMetaAnnotationFormula(platform);
     if (formula != null) {
-      prop.setSqlFormula(formula.select(), formula.join());
+      prop.setSqlFormula(processFormula(formula.select()), processFormula(formula.join()));
     }
 
     final Aggregation aggregation = prop.getMetaAnnotation(Aggregation.class);

ebean-core/src/main/java/io/ebeaninternal/server/deploy/parse/AnnotationParser.java

@@ -129,4 +129,11 @@ public abstract class AnnotationParser extends AnnotationBase {
     }
     return columnNames;
   }
+
+  /**
+   * Process any formula from @Formula or @Where.
+   */
+  protected String processFormula(String source) {
+    return source == null ? null : source.replace("${dbTableName}", descriptor.getBaseTable());
+  }
 }

ebean-core/src/main/java/io/ebeaninternal/server/expression/AbstractExpression.java

@@ -16,12 +16,17 @@ import io.ebeaninternal.api.NaturalKeyQueryData;
  */
 public abstract class AbstractExpression implements SpiExpression {
 
-  protected final String propName;
+  protected String propName;
 
   protected AbstractExpression(String propName) {
     this.propName = propName;
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    this.propName = path + "." + propName;
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // by default can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/expression/BetweenPropertyExpression.java

@@ -17,8 +17,8 @@ class BetweenPropertyExpression extends NonPrepareExpression {
 
   private static final String BETWEEN = " between ";
 
-  private final String lowProperty;
-  private final String highProperty;
+  private String lowProperty;
+  private String highProperty;
   private final Object value;
 
   BetweenPropertyExpression(String lowProperty, String highProperty, Object value) {
@@ -27,6 +27,12 @@ class BetweenPropertyExpression extends NonPrepareExpression {
     this.value = value;
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    this.lowProperty = path + "." + lowProperty;
+    this.highProperty = path + "." + highProperty;
+  }
+
   protected String name(String propName) {
     return propName;
   }

ebean-core/src/main/java/io/ebeaninternal/server/expression/DefaultExampleExpression.java

@@ -88,6 +88,13 @@ public class DefaultExampleExpression implements SpiExpression, ExampleExpressio
     }
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    for (SpiExpression exp : list) {
+      exp.prefixProperty(path);
+    }
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/expression/DefaultExpressionList.java

@@ -126,6 +126,13 @@ public class DefaultExpressionList<T> implements SpiExpressionList<T> {
     }
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    for (SpiExpression exp : list) {
+      exp.prefixProperty(path);
+    }
+  }
+
   @Override
   public Junction<T> toJunction() {
     return new JunctionExpression<>(Junction.Type.FILTER, this);

ebean-core/src/main/java/io/ebeaninternal/server/expression/ExistsQueryExpression.java

@@ -36,6 +36,11 @@ class ExistsQueryExpression implements SpiExpression, UnsupportedDocStoreExpress
     this.subQuery = null;
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    // do nothing
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/expression/IdExpression.java

@@ -19,6 +19,11 @@ class IdExpression extends NonPrepareExpression implements SpiExpression {
     this.value = value;
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    throw new IllegalStateException("Not allowed?");
+  }
+
   @Override
   public void writeDocQuery(DocQueryContext context) throws IOException {
     context.writeId(value);

ebean-core/src/main/java/io/ebeaninternal/server/expression/JunctionExpression.java

@@ -66,6 +66,11 @@ class JunctionExpression<T> implements SpiJunction<T>, SpiExpression, Expression
     this.exprList = exprList;
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    exprList.prefixProperty(path);
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/expression/LogicExpression.java

@@ -57,6 +57,12 @@ abstract class LogicExpression implements SpiExpression {
     this.expTwo = (SpiExpression) expTwo;
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    expOne.prefixProperty(path);
+    expTwo.prefixProperty(path);
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/expression/NestedPathWrapperExpression.java

@@ -24,6 +24,11 @@ class NestedPathWrapperExpression implements SpiExpression {
     this.delegate = delegate;
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    // do nothing
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/expression/NonPrepareExpression.java

@@ -9,6 +9,11 @@ import io.ebeaninternal.api.NaturalKeyQueryData;
  */
 abstract class NonPrepareExpression implements SpiExpression {
 
+  @Override
+  public void prefixProperty(String path) {
+    // do nothing
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/expression/NoopExpression.java

@@ -17,6 +17,11 @@ class NoopExpression implements SpiExpression {
 
   protected static final NoopExpression INSTANCE = new NoopExpression();
 
+  @Override
+  public void prefixProperty(String path) {
+    // do nothing
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/expression/NotExpression.java

@@ -22,6 +22,11 @@ final class NotExpression implements SpiExpression {
     this.exp = (SpiExpression) exp;
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    exp.prefixProperty(path);
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/lib/package.html

@@ -1,13 +0,0 @@
-<HTML>
-<HEAD>
-  <META HTTP-EQUIV="CONTENT-TYPE" CONTENT="text/html; charset=iso-8859-1">
-  <TITLE>AvajeLib</TITLE>
-</HEAD>
-<Body BGCOLOR="#ffffff">
-Core services for an application.
-<P>
-  Core services including Logging, Deployment properties and Background thread for
-  running frequent tasks.
-</P>
-</Body>
-</HTML>

ebean-core/src/main/java/io/ebeaninternal/server/loadcontext/DLoadManyContext.java

@@ -201,7 +201,6 @@ class DLoadManyContext extends DLoadBaseContext implements LoadManyContext {
 
     @Override
     public void loadMany(BeanCollection<?> bc, boolean onlyIds) {
-
       lock.lock();
       try {
         boolean useCache = !onlyIds && context.hitCache && context.property.isUseCache();
@@ -215,6 +214,7 @@ class DLoadManyContext extends DLoadBaseContext implements LoadManyContext {
               // find it using instance equality - avoiding equals() and potential deadlock issue
               if (list.get(i) == bc) {
                 list.remove(i);
+                bc.setLoader(context.parent.getEbeanServer());
                 return;
               }
             }
@@ -222,10 +222,9 @@ class DLoadManyContext extends DLoadBaseContext implements LoadManyContext {
           }
         }
 
-        // Should reduce the list by checking each beanCollection in the L2 first before executing the query
-
-        LoadManyRequest req = new LoadManyRequest(this, onlyIds, useCache);
-        context.parent.getEbeanServer().loadMany(req);
+        context.parent.getEbeanServer().loadMany(new LoadManyRequest(this, onlyIds, useCache));
+        // clear the buffer as all entries have been loaded
+        list.clear();
       } finally {
         lock.unlock();
       }

ebean-core/src/main/java/io/ebeaninternal/server/persist/dml/DmlHandler.java

@@ -3,7 +3,7 @@ package io.ebeaninternal.server.persist.dml;
 import io.ebeaninternal.api.SpiTransaction;
 import io.ebeaninternal.server.core.PersistRequestBean;
 import io.ebeaninternal.server.deploy.BeanProperty;
-import io.ebeaninternal.server.lib.Str;
+import io.ebeaninternal.server.util.Str;
 import io.ebeaninternal.server.persist.BatchedPstmt;
 import io.ebeaninternal.server.persist.BatchedPstmtHolder;
 import io.ebeaninternal.server.persist.dmlbind.BindableRequest;

ebean-core/src/main/java/io/ebeaninternal/server/query/CQuery.java

@@ -1,14 +1,9 @@
 package io.ebeaninternal.server.query;
 
+import io.ebean.CancelableQuery;
 import io.ebean.QueryIterator;
 import io.ebean.Version;
-import io.ebean.bean.BeanCollection;
-import io.ebean.bean.EntityBean;
-import io.ebean.bean.EntityBeanIntercept;
-import io.ebean.bean.NodeUsageCollector;
-import io.ebean.bean.NodeUsageListener;
-import io.ebean.bean.ObjectGraphNode;
-import io.ebean.bean.PersistenceContext;
+import io.ebean.bean.*;
 import io.ebean.core.type.DataReader;
 import io.ebean.event.readaudit.ReadEvent;
 import io.ebean.util.JdbcClose;
@@ -19,12 +14,7 @@ import io.ebeaninternal.api.SpiTransaction;
 import io.ebeaninternal.server.autotune.ProfilingListener;
 import io.ebeaninternal.server.core.OrmQueryRequest;
 import io.ebeaninternal.server.core.SpiOrmQueryRequest;
-import io.ebeaninternal.server.deploy.BeanCollectionHelpFactory;
-import io.ebeaninternal.server.deploy.BeanDescriptor;
-import io.ebeaninternal.server.deploy.BeanPropertyAssocMany;
-import io.ebeaninternal.server.deploy.BeanPropertyAssocOne;
-import io.ebeaninternal.server.deploy.DbReadContext;
-import io.ebeaninternal.server.type.DataBind;
+import io.ebeaninternal.server.deploy.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -34,11 +24,7 @@ import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.locks.ReentrantLock;
 
 /**
@@ -47,11 +33,9 @@ import java.util.concurrent.locks.ReentrantLock;
  * The SqlSelect is based on a tree (Object Graph). The tree is traversed to see
  * what parts are included in the tree according to the value of
  * find.getInclude();
- * </p>
  * <p>
  * The tree structure is flattened into a SqlSelectChain. The SqlSelectChain is
  * the key object used in reading the flat resultSet back into Objects.
- * </p>
  */
 public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTransactionEvent {
 
@@ -156,8 +140,6 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
    */
   private PreparedStatement pstmt;
 
-  private boolean cancelled;
-
   private String bindLog;
 
   private final CQueryPlan queryPlan;
@@ -205,19 +187,15 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
     this.query = request.getQuery();
     this.queryMode = query.getMode();
     this.lazyLoadManyProperty = query.getLazyLoadMany();
-
     this.readOnly = request.isReadOnly();
     this.disableLazyLoading = query.isDisableLazyLoading();
-
     this.objectGraphNode = query.getParentNode();
     this.profilingListener = query.getProfilingListener();
     this.autoTuneProfiling = profilingListener != null;
     this.profilingListenerRef = autoTuneProfiling ? new WeakReference<>(profilingListener) : null;
-
     // set the generated sql back to the query
     // so its available to the user...
     query.setGeneratedSql(queryPlan.getSql());
-
     SqlTree sqlTree = queryPlan.getSqlTree();
     this.rootNode = sqlTree.getRootNode();
     this.manyProperty = sqlTree.getManyProperty();
@@ -293,15 +271,7 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
   public void cancel() {
     lock.lock();
     try {
-      this.cancelled = true;
-      if (pstmt != null) {
-        try {
-          logger.debug("Cancelling query");
-          pstmt.cancel();
-        } catch (SQLException e) {
-          throw new PersistenceException("Error cancelling query", e);
-        }
-      }
+      JdbcClose.cancel(pstmt);
     } finally {
       lock.unlock();
     }
@@ -333,9 +303,8 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
   ResultSet prepareResultSet(boolean forwardOnlyHint) throws SQLException {
     lock.lock();
     try {
-      if (cancelled) {
-        throw new SQLException("Query cancelled");
-      }
+      // cancelled before we started
+      query.checkCancelled();
       startNano = System.nanoTime();
       SpiTransaction t = request.getTransaction();
       profileOffset = t.profileOffset();
@@ -363,17 +332,18 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
         pstmt.setFetchSize(query.getBufferFetchSizeHint());
       }
       bindLog = predicates.bind(queryPlan.bindEncryptedProperties(pstmt, conn));
-      return pstmt.executeQuery();
     } finally {
       lock.unlock();
     }
+    ResultSet ret = pstmt.executeQuery();
+    query.checkCancelled();
+    return ret;
   }
 
   /**
    * Close the resources.
    * <p>
    * The JDBC resultSet and statement need to be closed. Its important that this method is called.
-   * </p>
    */
   public void close() {
     try {
@@ -405,14 +375,12 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
 
   @Override
   public void setLazyLoadedChildBean(EntityBean bean, Object lazyLoadParentId) {
-
     if (lazyLoadParentId != null) {
       if (!lazyLoadParentId.equals(this.lazyLoadParentId)) {
         // get the appropriate parent bean from the persistence context
         this.lazyLoadParentBean = (EntityBean) lazyLoadManyProperty.getBeanDescriptor().contextGet(getPersistenceContext(), lazyLoadParentId);
         this.lazyLoadParentId = lazyLoadParentId;
       }
-
       // add the loadedBean to the appropriate collection of lazyLoadParentBean
       lazyLoadManyProperty.addBeanToCollectionWithCreate(lazyLoadParentBean, bean, true);
     }
@@ -423,10 +391,8 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
    * <p>
    * If the query includes a many then the first object in the returned array is
    * the one/master and the second the many/detail.
-   * </p>
    */
   private boolean readNextBean() throws SQLException {
-
     if (!moveToNextRow()) {
       if (currentBean == null) {
         nextBean = null;
@@ -440,7 +406,6 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
     }
 
     loadedBeanCount++;
-
     if (manyProperty == null) {
       // only single resultSet row required to build object so we are done
       // read a single resultSet row into single bean
@@ -486,14 +451,11 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
    * Return true if we can move to the next resultSet row.
    */
   private boolean moveToNextRow() throws SQLException {
-
     if (!dataReader.next()) {
       noMoreRows = true;
       return false;
     }
-
     rowCount++;
-    dataReader.resetColumnPosition();
     return true;
   }
 
@@ -502,7 +464,6 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
   }
 
   boolean readBean() throws SQLException {
-
     boolean result = hasNext();
     updateExecutionStatistics();
     return result;
@@ -522,7 +483,8 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
   boolean hasNext() throws SQLException {
     lock.lock();
     try {
-      if (noMoreRows || cancelled) {
+      query.checkCancelled();
+      if (noMoreRows) {
         return false;
       }
       if (hasNextCache) {
@@ -539,20 +501,16 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
    * Read version beans and their effective dates.
    */
   List<Version<T>> readVersions() throws SQLException {
-
     List<Version<T>> versionList = new ArrayList<>();
-
     Version<T> version;
     while ((version = readNextVersion()) != null) {
       versionList.add(version);
     }
-
     updateExecutionStatistics();
     return versionList;
   }
 
   private Version<T> readNextVersion() throws SQLException {
-
     if (moveToNextRow()) {
       return rootNode.loadVersion(this);
     }
@@ -696,7 +654,6 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
    * Should we create profileNodes for beans created in this query.
    * <p>
    * This is true for all queries except lazy load bean queries.
-   * </p>
    */
   @Override
   public boolean isAutoTuneProfiling() {
@@ -707,13 +664,11 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
   }
 
   private String getPath(String propertyName) {
-
     if (currentPrefix == null) {
       return propertyName;
     } else if (propertyName == null) {
       return currentPrefix;
     }
-
     String path = currentPathMap.get(propertyName);
     if (path != null) {
       return path;
@@ -724,7 +679,6 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
 
   @Override
   public void profileBean(EntityBeanIntercept ebi, String prefix) {
-
     ObjectGraphNode node = request.getGraphContext().getObjectGraphNode(prefix);
     ebi.setNodeUsageCollector(new NodeUsageCollector(node, profilingListenerRef));
   }
@@ -749,7 +703,6 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
    * a find many query with read auditing so build the ReadEvent and log it.
    */
   void auditFindMany() {
-
     if (auditIds != null && !auditIds.isEmpty()) {
       // get the id values of the underlying collection
       ReadEvent futureReadEvent = query.getFutureFetchAudit();
@@ -787,7 +740,6 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
    * Add the id to the audit id buffer and flush if needed in batches of 100.
    */
   private void auditNextBean() {
-
     if (auditIds == null) {
       auditIds = new ArrayList<>(100);
     }

ebean-core/src/main/java/io/ebeaninternal/server/query/CQueryBuilder.java

@@ -616,11 +616,19 @@ class CQueryBuilder {
         if (request.isInlineCountDistinct()) {
           sb.append(")");
         }
-        if (distinct && dbOrderBy != null && !query.isSingleAttribute()) {
+        if (distinct && dbOrderBy != null) {
           // add the orderBy columns to the select clause (due to distinct)
           final OrderBy<?> orderBy = query.getOrderBy();
           if (orderBy != null && orderBy.supportsSelect()) {
-            sb.append(", ").append(DbOrderByTrim.trim(dbOrderBy));
+            String trimmed = DbOrderByTrim.trim(dbOrderBy);
+            if (query.isSingleAttribute() && trimmed.equals(select.getSelectSql())) {
+              // NOP, already in SQL
+              // TODO: what to do if we select("id").orderBy("prop,id")?
+              // Can we live with a query like "select t0.id, t0.prop, t0.id from"
+              // or should we elliminate the second "t0.id" from select
+            } else {
+              sb.append(", ").append(trimmed);
+            }
           }
         }
       }

ebean-core/src/main/java/io/ebeaninternal/server/query/CQueryEngine.java

@@ -16,7 +16,7 @@ import io.ebeaninternal.server.core.DiffHelp;
 import io.ebeaninternal.server.core.OrmQueryRequest;
 import io.ebeaninternal.server.core.SpiResultSet;
 import io.ebeaninternal.server.deploy.BeanDescriptor;
-import io.ebeaninternal.server.lib.Str;
+import io.ebeaninternal.server.util.Str;
 import io.ebeaninternal.server.persist.Binder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -66,11 +66,13 @@ public class CQueryEngine {
 
   public <T> int delete(OrmQueryRequest<T> request) {
     CQueryUpdate query = queryBuilder.buildUpdateQuery(true, request);
+    request.setCancelableQuery(query);
     return executeUpdate(request, query);
   }
 
   public <T> int update(OrmQueryRequest<T> request) {
     CQueryUpdate query = queryBuilder.buildUpdateQuery(false, request);
+    request.setCancelableQuery(query);
     return executeUpdate(request, query);
   }
 
@@ -97,6 +99,7 @@ public class CQueryEngine {
   public <A> List<A> findSingleAttributeList(OrmQueryRequest<?> request) {
 
     CQueryFetchSingleAttribute rcQuery = queryBuilder.buildFetchAttributeQuery(request);
+    request.setCancelableQuery(rcQuery);
     return findAttributeList(request, rcQuery);
   }
 
@@ -151,6 +154,7 @@ public class CQueryEngine {
   public <A> List<A> findIds(OrmQueryRequest<?> request) {
 
     CQueryFetchSingleAttribute rcQuery = queryBuilder.buildFetchIdsQuery(request);
+    request.setCancelableQuery(rcQuery);
     return findAttributeList(request, rcQuery);
   }
 
@@ -164,6 +168,7 @@ public class CQueryEngine {
   public <T> int findCount(OrmQueryRequest<T> request) {
 
     CQueryRowCount rcQuery = queryBuilder.buildRowCountQuery(request);
+    request.setCancelableQuery(rcQuery);
     try {
 
       int count = rcQuery.findCount();
@@ -235,8 +240,10 @@ public class CQueryEngine {
 
     } catch (SQLException e) {
       try {
+        PersistenceException pex = cquery.createPersistenceException(e);
+        // create exception before closing connection
         cquery.close();
-        throw cquery.createPersistenceException(e);
+        throw pex;
       } finally {
         request.rollbackTransIfRequired();
       }
@@ -259,6 +266,7 @@ public class CQueryEngine {
     // order by lower sys period desc
     query.order().desc(sysPeriodLower);
     CQuery<T> cquery = queryBuilder.buildQuery(request);
+    request.setCancelableQuery(cquery);
     try {
       cquery.prepareBindExecuteQuery();
       if (request.isLogSql()) {
@@ -327,6 +335,7 @@ public class CQueryEngine {
    */
   public <T> SpiResultSet findResultSet(OrmQueryRequest<T> request) {
     CQuery<T> cquery = queryBuilder.buildQuery(request);
+    request.setCancelableQuery(cquery);
     try {
       boolean fwdOnly;
       if (request.isFindIterate()) {
@@ -411,6 +420,7 @@ public class CQueryEngine {
     EntityBean bean = null;
 
     CQuery<T> cquery = queryBuilder.buildQuery(request);
+    request.setCancelableQuery(cquery);
 
     try {
       cquery.prepareBindExecuteQuery();

ebean-core/src/main/java/io/ebeaninternal/server/query/CQueryFetchSingleAttribute.java

@@ -1,5 +1,6 @@
 package io.ebeaninternal.server.query;
 
+import io.ebean.CancelableQuery;
 import io.ebean.CountedValue;
 import io.ebean.core.type.ScalarDataReader;
 import io.ebean.util.JdbcClose;
@@ -18,11 +19,12 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.locks.ReentrantLock;
 
 /**
  * Base compiled query request for single attribute queries.
  */
-class CQueryFetchSingleAttribute implements SpiProfileTransactionEvent {
+class CQueryFetchSingleAttribute implements SpiProfileTransactionEvent, CancelableQuery {
 
   private static final Logger logger = LoggerFactory.getLogger(CQueryFetchSingleAttribute.class);
 
@@ -65,6 +67,8 @@ class CQueryFetchSingleAttribute implements SpiProfileTransactionEvent {
   private final boolean containsCounts;
 
   private long profileOffset;
+  
+  private final ReentrantLock lock = new ReentrantLock();
 
   /**
    * Create the Sql select based on the request.
@@ -111,7 +115,6 @@ class CQueryFetchSingleAttribute implements SpiProfileTransactionEvent {
           value = new CountedValue<>(value, dataReader.getLong());
         }
         result.add(value);
-        dataReader.resetColumnPosition();
         rowCount++;
       }
 
@@ -148,21 +151,27 @@ class CQueryFetchSingleAttribute implements SpiProfileTransactionEvent {
   }
 
   private void prepareExecute() throws SQLException {
-
-    SpiTransaction t = getTransaction();
-    profileOffset = t.profileOffset();
-    Connection conn = t.getInternalConnection();
-    pstmt = conn.prepareStatement(sql);
-
-    if (query.getBufferFetchSizeHint() > 0) {
-      pstmt.setFetchSize(query.getBufferFetchSizeHint());
+    lock.lock();
+    try {
+      query.checkCancelled();
+      SpiTransaction t = getTransaction();
+      profileOffset = t.profileOffset();
+      Connection conn = t.getInternalConnection();
+      pstmt = conn.prepareStatement(sql);
+  
+      if (query.getBufferFetchSizeHint() > 0) {
+        pstmt.setFetchSize(query.getBufferFetchSizeHint());
+      }
+      if (query.getTimeout() > 0) {
+        pstmt.setQueryTimeout(query.getTimeout());
+      }
+  
+      bindLog = predicates.bind(pstmt, conn);
+    } finally {
+      lock.unlock();
     }
-    if (query.getTimeout() > 0) {
-      pstmt.setQueryTimeout(query.getTimeout());
-    }
-
-    bindLog = predicates.bind(pstmt, conn);
     dataReader = new RsetDataReader(request.getDataTimeZone(), pstmt.executeQuery());
+    query.checkCancelled();
   }
 
   /**
@@ -195,4 +204,14 @@ class CQueryFetchSingleAttribute implements SpiProfileTransactionEvent {
   Set<String> getDependentTables() {
     return queryPlan.getDependentTables();
   }
+
+  @Override
+  public void cancel() {
+    lock.lock();
+    try {
+      JdbcClose.cancel(pstmt);
+    } finally {
+      lock.unlock();
+    }
+  }
 }

ebean-core/src/main/java/io/ebeaninternal/server/query/CQueryIteratorSimple.java

@@ -14,7 +14,7 @@ class CQueryIteratorSimple<T> implements QueryIterator<T> {
   private final CQuery<T> cquery;
 
   private final OrmQueryRequest<T> request;
-  
+
   private boolean closed;
 
   CQueryIteratorSimple(CQuery<T> cquery, OrmQueryRequest<T> request) {
@@ -54,8 +54,4 @@ class CQueryIteratorSimple<T> implements QueryIterator<T> {
     }
   }
 
-  @Override
-  public void remove() {
-    throw new PersistenceException("Remove not allowed");
-  }
 }

ebean-core/src/main/java/io/ebeaninternal/server/query/CQueryIteratorWithBuffer.java

@@ -76,8 +76,4 @@ class CQueryIteratorWithBuffer<T> implements QueryIterator<T> {
     }
   }
 
-  @Override
-  public void remove() {
-    throw new PersistenceException("Remove not allowed");
-  }
 }

ebean-core/src/main/java/io/ebeaninternal/server/query/CQueryPlan.java

@@ -13,7 +13,7 @@ import io.ebeaninternal.api.SpiQueryBindCapture;
 import io.ebeaninternal.api.SpiQueryPlan;
 import io.ebeaninternal.server.core.OrmQueryRequest;
 import io.ebeaninternal.server.core.timezone.DataTimeZone;
-import io.ebeaninternal.server.lib.Str;
+import io.ebeaninternal.server.util.Str;
 import io.ebeaninternal.server.query.CQueryPlanStats.Snapshot;
 import io.ebeaninternal.server.type.DataBind;
 import io.ebeaninternal.server.type.DataBindCapture;

ebean-core/src/main/java/io/ebeaninternal/server/query/CQueryRowCount.java

@@ -1,5 +1,6 @@
 package io.ebeaninternal.server.query;
 
+import io.ebean.CancelableQuery;
 import io.ebean.util.JdbcClose;
 import io.ebeaninternal.api.SpiProfileTransactionEvent;
 import io.ebeaninternal.api.SpiQuery;
@@ -13,11 +14,12 @@ import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.Set;
+import java.util.concurrent.locks.ReentrantLock;
 
 /**
  * Executes the select row count query.
  */
-class CQueryRowCount implements SpiProfileTransactionEvent {
+class CQueryRowCount implements SpiProfileTransactionEvent, CancelableQuery {
 
   private final CQueryPlan queryPlan;
 
@@ -57,6 +59,8 @@ class CQueryRowCount implements SpiProfileTransactionEvent {
   private int rowCount;
 
   private long profileOffset;
+  
+  private final ReentrantLock lock = new ReentrantLock();
 
   /**
    * Create the Sql select based on the request.
@@ -110,14 +114,22 @@ class CQueryRowCount implements SpiProfileTransactionEvent {
       SpiTransaction t = getTransaction();
       profileOffset = t.profileOffset();
       Connection conn = t.getInternalConnection();
-      pstmt = conn.prepareStatement(sql);
+      lock.lock();
+      try {
+        query.checkCancelled();
+        pstmt = conn.prepareStatement(sql);
 
-      if (query.getTimeout() > 0) {
-        pstmt.setQueryTimeout(query.getTimeout());
+        if (query.getTimeout() > 0) {
+          pstmt.setQueryTimeout(query.getTimeout());
+        }
+
+        bindLog = predicates.bind(pstmt, conn);
+      } finally {
+        lock.unlock();
       }
-
-      bindLog = predicates.bind(pstmt, conn);
       rset = pstmt.executeQuery();
+      query.checkCancelled();
+      
       if (!rset.next()) {
         throw new PersistenceException("Expecting 1 row but got none?");
       }
@@ -161,4 +173,14 @@ class CQueryRowCount implements SpiProfileTransactionEvent {
   Set<String> getDependentTables() {
     return queryPlan.getDependentTables();
   }
+
+  @Override
+  public void cancel() {
+    lock.lock();
+    try {
+      JdbcClose.cancel(pstmt);
+    } finally {
+      lock.unlock();
+    }
+  }
 }

ebean-core/src/main/java/io/ebeaninternal/server/query/CQueryUpdate.java

@@ -1,5 +1,6 @@
 package io.ebeaninternal.server.query;
 
+import io.ebean.CancelableQuery;
 import io.ebean.util.JdbcClose;
 import io.ebeaninternal.api.SpiProfileTransactionEvent;
 import io.ebeaninternal.api.SpiQuery;
@@ -10,11 +11,12 @@ import io.ebeaninternal.server.deploy.BeanDescriptor;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import java.util.concurrent.locks.ReentrantLock;
 
 /**
- * Executes the delete query.
+ * Executes the update query.
  */
-class CQueryUpdate implements SpiProfileTransactionEvent {
+class CQueryUpdate implements SpiProfileTransactionEvent, CancelableQuery {
 
   private final CQueryPlan queryPlan;
 
@@ -45,6 +47,8 @@ class CQueryUpdate implements SpiProfileTransactionEvent {
 
   private long profileOffset;
 
+  private final ReentrantLock lock = new ReentrantLock();
+  
   /**
    * Create the Sql select based on the request.
    */
@@ -82,15 +86,22 @@ class CQueryUpdate implements SpiProfileTransactionEvent {
       SpiTransaction t = getTransaction();
       profileOffset = t.profileOffset();
       Connection conn = t.getInternalConnection();
-      pstmt = conn.prepareStatement(sql);
+      lock.lock();
+      try {
+        query.checkCancelled();
+        pstmt = conn.prepareStatement(sql);
 
-      if (query.getTimeout() > 0) {
-        pstmt.setQueryTimeout(query.getTimeout());
+        if (query.getTimeout() > 0) {
+          pstmt.setQueryTimeout(query.getTimeout());
+        }
+
+        bindLog = predicates.bind(pstmt, conn);
+      } finally {
+        lock.unlock();
       }
-
-      bindLog = predicates.bind(pstmt, conn);
       rowCount = pstmt.executeUpdate();
-
+      query.checkCancelled();
+      
       long executionTimeMicros = (System.nanoTime() - startNano) / 1000L;
       request.slowQueryCheck(executionTimeMicros, rowCount);
       if (queryPlan.executionTime(executionTimeMicros)) {
@@ -122,4 +133,14 @@ class CQueryUpdate implements SpiProfileTransactionEvent {
       .profileStream()
       .addQueryEvent(query.profileEventId(), profileOffset, desc.getName(), rowCount, query.getProfileId());
   }
+
+  @Override
+  public void cancel() {
+    lock.lock();
+    try {
+      JdbcClose.cancel(pstmt);
+    } finally {
+      lock.unlock();
+    }
+  }
 }

ebean-core/src/main/java/io/ebeaninternal/server/query/DefaultDbSqlContext.java

@@ -11,11 +11,10 @@ import java.util.HashSet;
 class DefaultDbSqlContext implements DbSqlContext {
 
   private static final String COMMA = ", ";
-
   private static final String PERIOD = ".";
   private static final int STRING_BUILDER_INITIAL_CAPACITY = 140;
-
   private static final String tableAliasPlaceHolder = "${ta}";
+  private static final String tableAliasManyPlaceHolder = "${mta}";
 
   private final String columnAliasPrefix;
 
@@ -111,27 +110,22 @@ class DefaultDbSqlContext implements DbSqlContext {
   }
 
   @Override
-  public void addJoin(String type, String table, TableJoinColumn[] cols, String a1, String a2) {
-
+  public void addJoin(String type, String table, TableJoinColumn[] cols, String a1, String a2, String extraWhere) {
     if (tableJoins == null) {
       tableJoins = new HashSet<>();
     }
-
     String joinKey = table + "-" + a1 + "-" + a2;
     if (tableJoins.contains(joinKey)) {
       return;
     }
 
     tableJoins.add(joinKey);
-
     sb.append(" ").append(type);
     boolean addAsOfOnClause = false;
     if (draftSupport != null) {
       appendTable(table, draftSupport.getDraftTable(table));
-
     } else if (!historyQuery) {
       sb.append(" ").append(table).append(" ");
-
     } else {
       // check if there is an associated history table and if so
       // use the unionAll view - we expect an additional predicate to match
@@ -164,13 +158,17 @@ class DefaultDbSqlContext implements DbSqlContext {
     if (addAsOfOnClause) {
       sb.append(" and ").append(historySupport.getAsOfPredicate(a2));
     }
+    if (extraWhere != null && !extraWhere.isEmpty()) {
+      sb.append(" and ");
+      // we will also need a many-table alias here
+      sb.append(extraWhere.replace(tableAliasPlaceHolder, a2).replace(tableAliasManyPlaceHolder, a1));
+    }
   }
 
   private void appendTable(String table, String draftTable) {
     if (draftTable != null) {
       // there is an associated history table and view so use that
       sb.append(" ").append(draftTable).append(" ");
-
     } else {
       sb.append(" ").append(table).append(" ");
     }
@@ -193,7 +191,6 @@ class DefaultDbSqlContext implements DbSqlContext {
 
   @Override
   public String getRelativePrefix(String propName) {
-
     return currentPrefix == null ? propName : currentPrefix + "." + propName;
   }
 
@@ -228,7 +225,6 @@ class DefaultDbSqlContext implements DbSqlContext {
       // the same join has already been added.
       return;
     }
-
     // we only want to add this join once
     formulaJoins.add(converted);
     sb.append(" ");
@@ -263,13 +259,10 @@ class DefaultDbSqlContext implements DbSqlContext {
 
   @Override
   public void appendHistorySysPeriod() {
-
     String tableAlias = tableAliasStack.peek();
-
     sb.append(COMMA);
     sb.append(historySupport.getSysPeriodLower(tableAlias));
     appendColumnAlias();
-
     sb.append(COMMA);
     sb.append(historySupport.getSysPeriodUpper(tableAlias));
     appendColumnAlias();

ebean-core/src/main/java/io/ebeaninternal/server/query/DefaultRelationalQueryEngine.java

@@ -3,6 +3,7 @@ package io.ebeaninternal.server.query;
 import io.ebean.RowConsumer;
 import io.ebean.RowMapper;
 import io.ebean.SqlRow;
+import io.ebean.core.type.DataReader;
 import io.ebean.core.type.ScalarType;
 import io.ebean.meta.MetricVisitor;
 import io.ebean.metric.MetricFactory;
@@ -10,10 +11,10 @@ import io.ebean.metric.TimedMetricMap;
 import io.ebeaninternal.api.SpiQuery;
 import io.ebeaninternal.server.core.RelationalQueryEngine;
 import io.ebeaninternal.server.core.RelationalQueryRequest;
+import io.ebeaninternal.server.core.RowReader;
 import io.ebeaninternal.server.persist.Binder;
 
 import javax.persistence.PersistenceException;
-import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Consumer;
@@ -59,12 +60,26 @@ public class DefaultRelationalQueryEngine implements RelationalQueryEngine {
   }
 
   @Override
-  public void findEach(RelationalQueryRequest request, Predicate<SqlRow> consumer) {
+  public void findEach(RelationalQueryRequest request, RowConsumer consumer) {
+    try {
+      request.executeSql(binder, SpiQuery.Type.ITERATE);
+      request.mapEach(consumer);
+      request.logSummary();
 
+    } catch (Exception e) {
+      throw new PersistenceException(errMsg(e.getMessage(), request.getSql()), e);
+
+    } finally {
+      request.close();
+    }
+  }
+
+  @Override
+  public <T> void findEach(RelationalQueryRequest request, RowReader<T> reader, Predicate<T> consumer) {
     try {
       request.executeSql(binder, SpiQuery.Type.ITERATE);
       while (request.next()) {
-        if (!consumer.test(readRow(request))) {
+        if (!consumer.test(reader.read())) {
           break;
         }
       }
@@ -79,25 +94,7 @@ public class DefaultRelationalQueryEngine implements RelationalQueryEngine {
   }
 
   @Override
-  public void findEach(RelationalQueryRequest request, Consumer<SqlRow> consumer) {
-
-    try {
-      request.executeSql(binder, SpiQuery.Type.ITERATE);
-      while (request.next()) {
-        consumer.accept(readRow(request));
-      }
-      request.logSummary();
-
-    } catch (Exception e) {
-      throw new PersistenceException(errMsg(e.getMessage(), request.getSql()), e);
-
-    } finally {
-      request.close();
-    }
-  }
-
-  @Override
-  public <T> T findOneMapper(RelationalQueryRequest request, RowMapper<T> mapper) {
+  public <T> T findOne(RelationalQueryRequest request, RowMapper<T> mapper) {
     try {
       request.executeSql(binder, SpiQuery.Type.BEAN);
       T value = request.mapOne(mapper);
@@ -113,13 +110,15 @@ public class DefaultRelationalQueryEngine implements RelationalQueryEngine {
   }
 
   @Override
-  public <T> List<T> findListMapper(RelationalQueryRequest request, RowMapper<T> mapper) {
+  public <T> List<T> findList(RelationalQueryRequest request, RowReader<T> reader) {
     try {
       request.executeSql(binder, SpiQuery.Type.LIST);
-      List<T> list = request.mapList(mapper);
+      List<T> rows = new ArrayList<>();
+      while (request.next()) {
+        rows.add(reader.read());
+      }
       request.logSummary();
-      return list;
-
+      return rows;
     } catch (Exception e) {
       throw new PersistenceException(errMsg(e.getMessage(), request.getSql()), e);
 
@@ -128,12 +127,19 @@ public class DefaultRelationalQueryEngine implements RelationalQueryEngine {
     }
   }
 
+  @SuppressWarnings("unchecked")
   @Override
-  public void findEachRow(RelationalQueryRequest request, RowConsumer consumer) {
+  public <T> T findSingleAttribute(RelationalQueryRequest request, Class<T> cls) {
+    ScalarType<T> scalarType = (ScalarType<T>) binder.getScalarType(cls);
     try {
-      request.executeSql(binder, SpiQuery.Type.LIST);
-      request.mapEach(consumer);
+      request.executeSql(binder, SpiQuery.Type.ATTRIBUTE);
+      final DataReader dataReader = binder.createDataReader(request.getResultSet());
+      T value = null;
+      if (dataReader.next()) {
+        value = scalarType.read(dataReader);
+      }
       request.logSummary();
+      return value;
 
     } catch (Exception e) {
       throw new PersistenceException(errMsg(e.getMessage(), request.getSql()), e);
@@ -147,68 +153,13 @@ public class DefaultRelationalQueryEngine implements RelationalQueryEngine {
   @Override
   public <T> List<T> findSingleAttributeList(RelationalQueryRequest request, Class<T> cls) {
     ScalarType<T> scalarType = (ScalarType<T>) binder.getScalarType(cls);
-    return findScalarList(request, scalarType);
-  }
-
-  private <T> List<T> findScalarList(RelationalQueryRequest request, ScalarType<T> scalarType) {
     try {
       request.executeSql(binder, SpiQuery.Type.ATTRIBUTE);
-      List<T> list = new ArrayList<>();
-      while (request.next()) {
-        request.incrementRows();
-        list.add(scalarType.read(binder.createDataReader(request.getResultSet())));
+      final DataReader dataReader = binder.createDataReader(request.getResultSet());
+      List<T> rows = new ArrayList<>();
+      while (dataReader.next()) {
+        rows.add(scalarType.read(dataReader));
       }
-
-      request.logSummary();
-      return list;
-
-    } catch (Exception e) {
-      throw new PersistenceException(errMsg(e.getMessage(), request.getSql()), e);
-
-    } finally {
-      request.close();
-    }
-  }
-
-  @SuppressWarnings("unchecked")
-  @Override
-  public <T> T findSingleAttribute(RelationalQueryRequest request, Class<T> cls) {
-
-    ScalarType<T> scalarType = (ScalarType<T>) binder.getScalarType(cls);
-    return findScalar(request, scalarType);
-  }
-
-  private <T> T findScalar(RelationalQueryRequest request, ScalarType<T> scalarType) {
-    try {
-      request.executeSql(binder, SpiQuery.Type.ATTRIBUTE);
-
-      T value = null;
-      if (request.next()) {
-        request.incrementRows();
-        value = scalarType.read(binder.createDataReader(request.getResultSet()));
-      }
-
-      request.logSummary();
-      return value;
-
-    } catch (Exception e) {
-      throw new PersistenceException(errMsg(e.getMessage(), request.getSql()), e);
-
-    } finally {
-      request.close();
-    }
-  }
-
-  @Override
-  public List<SqlRow> findList(RelationalQueryRequest request) {
-
-    try {
-      request.executeSql(binder, SpiQuery.Type.LIST);
-      List<SqlRow> rows = new ArrayList<>();
-      while (request.next()) {
-        rows.add(readRow(request));
-      }
-
       request.logSummary();
       return rows;
 
@@ -220,11 +171,23 @@ public class DefaultRelationalQueryEngine implements RelationalQueryEngine {
     }
   }
 
-  /**
-   * Read the row from the ResultSet and return as a MapBean.
-   */
-  private SqlRow readRow(RelationalQueryRequest request) throws SQLException {
-    return request.createNewRow();
-  }
+  @SuppressWarnings("unchecked")
+  @Override
+  public <T> void findSingleAttributeEach(RelationalQueryRequest request, Class<T> cls, Consumer<T> consumer) {
+    ScalarType<T> scalarType = (ScalarType<T>) binder.getScalarType(cls);
+    try {
+      request.executeSql(binder, SpiQuery.Type.ATTRIBUTE);
+      final DataReader dataReader = binder.createDataReader(request.getResultSet());
+      while (dataReader.next()) {
+        consumer.accept(scalarType.read(dataReader));
+      }
+      request.logSummary();
 
+    } catch (Exception e) {
+      throw new PersistenceException(errMsg(e.getMessage(), request.getSql()), e);
+
+    } finally {
+      request.close();
+    }
+  }
 }

ebean-core/src/main/java/io/ebeaninternal/server/query/DtoQueryEngine.java

@@ -1,10 +1,12 @@
 package io.ebeaninternal.server.query;
 
+import io.ebean.QueryIterator;
 import io.ebeaninternal.api.SpiQuery;
 import io.ebeaninternal.server.core.DtoQueryRequest;
 import io.ebeaninternal.server.persist.Binder;
 
 import javax.persistence.PersistenceException;
+import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Consumer;
@@ -27,20 +29,29 @@ public class DtoQueryEngine {
       }
       return rows;
 
-    } catch (Throwable e) {
+    } catch (SQLException e) {
       throw new PersistenceException(errMsg(e.getMessage(), request.getSql()), e);
     } finally {
       request.close();
     }
   }
 
+  public <T> QueryIterator<T> findIterate(DtoQueryRequest<T> request) {
+    try {
+      request.executeSql(binder, SpiQuery.Type.ITERATE);
+      return new DtoQueryIterator<>(request);
+    } catch (SQLException e) {
+      throw new PersistenceException(errMsg(e.getMessage(), request.getSql()), e);
+    }
+  }
+
   public <T> void findEach(DtoQueryRequest<T> request, Consumer<T> consumer) {
      try {
       request.executeSql(binder, SpiQuery.Type.ITERATE);
       while (request.next()) {
         consumer.accept(request.readNextBean());
       }
-    } catch (Exception e) {
+    } catch (SQLException e) {
       throw new PersistenceException(errMsg(e.getMessage(), request.getSql()), e);
     } finally {
       request.close();
@@ -77,9 +88,8 @@ public class DtoQueryEngine {
           break;
         }
       }
-    } catch (Exception e) {
+    } catch (SQLException e) {
       throw new PersistenceException(errMsg(e.getMessage(), request.getSql()), e);
-
     } finally {
       request.close();
     }

ebean-core/src/main/java/io/ebeaninternal/server/query/DtoQueryIterator.java

@@ -0,0 +1,51 @@
+package io.ebeaninternal.server.query;
+
+import io.ebean.QueryIterator;
+import io.ebeaninternal.server.core.DtoQueryRequest;
+
+import javax.persistence.PersistenceException;
+import java.sql.SQLException;
+
+class DtoQueryIterator<T> implements QueryIterator<T> {
+
+  private final DtoQueryRequest<T> request;
+  private boolean closed;
+
+  DtoQueryIterator(DtoQueryRequest<T> request) {
+    this.request = request;
+  }
+
+  @Override
+  public boolean hasNext() {
+    boolean result = false;
+    try {
+      result = request.next();
+      return result;
+    } catch (SQLException e) {
+      throw new PersistenceException(e);
+    } finally {
+      if (!result) {
+        close();
+      }
+    }
+  }
+
+  @Override
+  public T next() {
+    try {
+      return request.readNextBean();
+    } catch (SQLException e) {
+      throw new PersistenceException(e);
+    }
+  }
+
+  @Override
+  public void close() {
+    if (!closed) {
+      closed = true;
+      request.close();
+      request.endTransIfRequired();
+    }
+  }
+
+}

ebean-core/src/main/java/io/ebeaninternal/server/query/SqlTreeNodeExtraJoin.java

@@ -5,6 +5,7 @@ import io.ebean.bean.EntityBean;
 import io.ebean.core.type.ScalarType;
 import io.ebean.util.SplitName;
 import io.ebeaninternal.api.SpiQuery;
+import io.ebeaninternal.server.deploy.BeanPropertyAssocOne;
 import io.ebeaninternal.server.deploy.DbReadContext;
 import io.ebeaninternal.server.deploy.DbSqlContext;
 import io.ebeaninternal.server.deploy.TableJoin;
@@ -130,6 +131,14 @@ class SqlTreeNodeExtraJoin implements SqlTreeNode {
       }
     }
 
+    boolean oneToOneExported = false;
+    if (assocBeanProperty instanceof BeanPropertyAssocOne) {
+      BeanPropertyAssocOne<?> oneToOneProp = (BeanPropertyAssocOne<?>) assocBeanProperty;
+      if (oneToOneProp.isOneToOneExported()) {
+        oneToOneExported = true;
+      }
+    }
+
     if (pathContainsMany) {
       // "promote" to left join as the path contains a many
       joinType = SqlJoinType.OUTER;
@@ -140,7 +149,7 @@ class SqlTreeNodeExtraJoin implements SqlTreeNode {
         assocBeanProperty.appendFrom(ctx, joinType);
       }
       joinType = assocBeanProperty.addJoin(joinType, prefix, ctx);
-      if (assocBeanProperty.isTargetSoftDelete() && temporalMode != SpiQuery.TemporalMode.SOFT_DELETED) {
+      if (!oneToOneExported && assocBeanProperty.isTargetSoftDelete() && temporalMode != SpiQuery.TemporalMode.SOFT_DELETED) {
         ctx.append(" and ").append(assocBeanProperty.getSoftDeletePredicate(ctx.getTableAlias(prefix)));
       }
     }

ebean-core/src/main/java/io/ebeaninternal/server/query/SqlTreeNodeManyRoot.java

@@ -47,16 +47,6 @@ final class SqlTreeNodeManyRoot extends SqlTreeNodeBean {
     ctx.popTableAlias();
   }
 
-  /**
-   * append extraWhere to the join.
-   */
-  @Override
-  protected SqlJoinType appendFromAsJoin(DbSqlContext ctx, SqlJoinType joinType) {
-    SqlJoinType join = super.appendFromAsJoin(ctx, joinType);
-    super.appendExtraWhere(ctx);
-    return join;
-  }
-
   @Override
   protected void appendExtraWhere(DbSqlContext ctx) {
     // extraWhere is already appended to the tableJoin

ebean-core/src/main/java/io/ebeaninternal/server/querydefn/AbstractQuery.java

@@ -0,0 +1,56 @@
+package io.ebeaninternal.server.querydefn;
+
+import java.util.concurrent.locks.ReentrantLock;
+
+import javax.persistence.PersistenceException;
+
+import io.ebean.CancelableQuery;
+import io.ebeaninternal.api.SpiCancelableQuery;
+
+/**
+ * Common code for Dto/Orm/RelationalQuery
+ *
+ * @author Roland Praml, FOCONIS AG
+ *
+ */
+public class AbstractQuery implements SpiCancelableQuery {
+
+  private boolean cancelled;
+
+  private CancelableQuery cancelableQuery;
+
+  private final ReentrantLock lock = new ReentrantLock();
+  
+  @Override
+  public void cancel() {
+    lock.lock();
+    try {
+      if (!cancelled) {
+        cancelled = true;
+        if (cancelableQuery != null) {
+          cancelableQuery.cancel();
+        }
+      }
+    } finally {
+      lock.unlock();
+    }
+  }
+
+  @Override
+  public void checkCancelled() {
+    if (cancelled) {
+      throw new PersistenceException("Query was cancelled");
+    }
+  }
+
+  @Override
+  public void setCancelableQuery(CancelableQuery cancelableQuery) {
+    lock.lock();
+    try {
+      checkCancelled();
+      this.cancelableQuery = cancelableQuery;
+    } finally {
+      lock.unlock();
+    }
+  }
+}

ebean-core/src/main/java/io/ebeaninternal/server/querydefn/DefaultDtoQuery.java

@@ -2,6 +2,7 @@ package io.ebeaninternal.server.querydefn;
 
 import io.ebean.DtoQuery;
 import io.ebean.ProfileLocation;
+import io.ebean.QueryIterator;
 import io.ebean.Transaction;
 import io.ebeaninternal.api.BindParams;
 import io.ebeaninternal.api.SpiDtoQuery;
@@ -15,11 +16,12 @@ import java.util.List;
 import java.util.Optional;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
+import java.util.stream.Stream;
 
 /**
  * Default implementation of DtoQuery.
  */
-public class DefaultDtoQuery<T> implements SpiDtoQuery<T> {
+public class DefaultDtoQuery<T> extends AbstractQuery implements SpiDtoQuery<T> {
 
   private final SpiEbeanServer server;
 
@@ -112,6 +114,16 @@ public class DefaultDtoQuery<T> implements SpiDtoQuery<T> {
     server.findDtoEachWhile(this, consumer);
   }
 
+  @Override
+  public QueryIterator<T> findIterate() {
+    return server.findDtoIterate(this);
+  }
+
+  @Override
+  public Stream<T> findStream() {
+    return server.findDtoStream(this);
+  }
+
   @Override
   public List<T> findList() {
     return server.findDtoList(this);

ebean-core/src/main/java/io/ebeaninternal/server/querydefn/DefaultOrmQuery.java

@@ -55,10 +55,10 @@ import io.ebeaninternal.server.deploy.BeanNaturalKey;
 import io.ebeaninternal.server.deploy.BeanPropertyAssocMany;
 import io.ebeaninternal.server.deploy.InheritInfo;
 import io.ebeaninternal.server.deploy.TableJoin;
+import io.ebeaninternal.server.el.ElPropertyDeploy;
 import io.ebeaninternal.server.expression.DefaultExpressionList;
 import io.ebeaninternal.server.expression.IdInExpression;
 import io.ebeaninternal.server.expression.SimpleExpression;
-import io.ebeaninternal.server.query.CancelableQuery;
 import io.ebeaninternal.server.query.NativeSqlQueryPlanKey;
 import io.ebeaninternal.server.rawsql.SpiRawSql;
 import io.ebeaninternal.server.transaction.ExternalJdbcTransaction;
@@ -81,7 +81,7 @@ import java.util.stream.Stream;
 /**
  * Default implementation of an Object Relational query.
  */
-public class DefaultOrmQuery<T> implements SpiQuery<T> {
+public class DefaultOrmQuery<T> extends AbstractQuery implements SpiQuery<T> {
 
   private static final String DEFAULT_QUERY_NAME = "default";
 
@@ -113,10 +113,6 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
 
   private ProfilingListener profilingListener;
 
-  private boolean cancelled;
-
-  private CancelableQuery cancelableQuery;
-
   private Type type;
 
   private String label;
@@ -539,6 +535,14 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
     if (havingExpressions != null) {
       havingExpressions.containsMany(beanDescriptor, manyWhereJoins);
     }
+    if (orderBy != null) {
+      for (Property orderProperty : orderBy.getProperties()) {
+        ElPropertyDeploy elProp = beanDescriptor.getElPropertyDeploy(orderProperty.getProperty());
+        if (elProp != null && elProp.containsFormulaWithJoin()) {
+          manyWhereJoins.addFormulaWithJoin(orderProperty.getProperty());
+        }
+      }
+    }
   }
 
   /**
@@ -856,6 +860,7 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
     copy.parentNode = parentNode;
     copy.forUpdate = forUpdate;
     copy.rawSql = rawSql;
+    setCancelableQuery(copy); // required to cancel findId query
     return copy;
   }
 
@@ -1238,6 +1243,7 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
    * Prepare the expressions (compile sub-queries etc).
    */
   private void prepareExpressions(BeanQueryRequest<?> request) {
+    detail.prepareExpressions(request);
     if (whereExpressions != null) {
       whereExpressions.prepareExpression(request);
     }
@@ -2045,16 +2051,6 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
     return futureFetchAudit;
   }
 
-  @Override
-  public void setCancelableQuery(CancelableQuery cancelableQuery) {
-    lock.lock();
-    try {
-      this.cancelableQuery = cancelableQuery;
-    } finally {
-      lock.unlock();
-    }
-  }
-
   @Override
   public Query<T> setBaseTable(String baseTable) {
     this.baseTable = baseTable;
@@ -2082,28 +2078,6 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
     return rootTableAlias != null ? rootTableAlias : defaultAlias;
   }
 
-  @Override
-  public void cancel() {
-    lock.lock();
-    try {
-      if (!cancelled && cancelableQuery != null) {
-        cancelled = true;
-        cancelableQuery.cancel();
-      }
-    } finally {
-      lock.unlock();
-    }
-  }
-
-  @Override
-  public boolean isCancelled() {
-    lock.lock();
-    try {
-      return cancelled;
-    } finally {
-      lock.unlock();
-    }
-  }
 
   @Override
   public Set<String> validate() {

ebean-core/src/main/java/io/ebeaninternal/server/querydefn/DefaultRelationalQuery.java

@@ -17,15 +17,15 @@ import java.util.function.Predicate;
 /**
  * Default implementation of SQuery - SQL Query.
  */
-public class DefaultRelationalQuery implements SpiSqlQuery {
+public class DefaultRelationalQuery extends AbstractQuery implements SpiSqlQuery {
 
   private static final long serialVersionUID = -1098305779779591068L;
 
   private final transient SpiEbeanServer server;
 
-  private String label;
+  private final String query;
 
-  private String query;
+  private String label;
 
   private int firstRow;
 
@@ -208,19 +208,25 @@ public class DefaultRelationalQuery implements SpiSqlQuery {
     return query;
   }
 
-  <T> T mapperFindOne(RowMapper<T> mapper) {
+  private <T> T mapperFindOne(RowMapper<T> mapper) {
     return server.findOneMapper(this, mapper);
   }
 
-  <T> List<T> mapperFindList(RowMapper<T> mapper) {
+  private <T> List<T> mapperFindList(RowMapper<T> mapper) {
     return server.findListMapper(this, mapper);
   }
 
+  private <T> void mapperFindEach(RowMapper<T> mapper, Consumer<T> consumer) {
+    server.findEachRow(this, (resultSet, rowNum) -> consumer.accept(mapper.map(resultSet, rowNum)));
+  }
+
+  @SuppressWarnings({"rawtypes", "unchecked"})
   @Override
   public <T> TypeQuery<T> mapToScalar(Class<T> attributeType) {
     return new Scalar(attributeType);
   }
 
+  @SuppressWarnings({"rawtypes", "unchecked"})
   @Override
   public <T> TypeQuery<T> mapTo(RowMapper<T> mapper) {
     return new Mapper(mapper);
@@ -248,6 +254,15 @@ public class DefaultRelationalQuery implements SpiSqlQuery {
     public List<T> findList() {
       return findSingleAttributeList(type);
     }
+
+    @Override
+    public void findEach(Consumer<T> consumer) {
+      scalarFindEach(type, consumer);
+    }
+  }
+
+  private <T> void scalarFindEach(Class<T> type, Consumer<T> consumer) {
+    server.findSingleAttributeEach(this, type, consumer);
   }
 
   private class Mapper<T> implements SqlQuery.TypeQuery<T> {
@@ -272,5 +287,10 @@ public class DefaultRelationalQuery implements SpiSqlQuery {
     public List<T> findList() {
       return mapperFindList(mapper);
     }
+
+    @Override
+    public void findEach(Consumer<T> consumer) {
+      mapperFindEach(mapper, consumer);
+    }
   }
 }

ebean-core/src/main/java/io/ebeaninternal/server/querydefn/OrmQueryDetail.java

@@ -1,6 +1,7 @@
 package io.ebeaninternal.server.querydefn;
 
 import io.ebean.FetchConfig;
+import io.ebean.event.BeanQueryRequest;
 import io.ebean.util.SplitName;
 import io.ebeaninternal.server.deploy.BeanDescriptor;
 import io.ebeaninternal.server.deploy.BeanPropertyAssoc;
@@ -9,25 +10,16 @@ import io.ebeaninternal.server.el.ElPropertyValue;
 
 import javax.persistence.PersistenceException;
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 
 /**
  * Represents the internal structure of an Object Relational query.
  * <p>
  * Holds the select() and join() details of a ORM query.
- * </p>
  * <p>
  * It is worth noting that for AutoTune a "tuned fetch info" builds an instance of OrmQueryDetail.
  * Tuning a query is a matter of replacing an instance of this class with one that has been tuned
  * with select() and join() set.
- * </p>
  */
 public class OrmQueryDetail implements Serializable {
 
@@ -81,7 +73,6 @@ public class OrmQueryDetail implements Serializable {
    * Return true if equal in terms of autoTune (select and fetch without property ordering).
    */
   public boolean isAutoTuneEqual(OrmQueryDetail otherDetail) {
-
     if (!isSameByAutoTune(baseProps, otherDetail.baseProps)) {
       return false;
     }
@@ -109,7 +100,7 @@ public class OrmQueryDetail implements Serializable {
   /**
    * Return the detail in string form.
    */
-  public String asStringDebug() {
+  public String asString() {
     StringBuilder sb = new StringBuilder();
     if (baseProps.hasProperties()) {
       baseProps.asStringDebug("select ", sb);
@@ -175,9 +166,7 @@ public class OrmQueryDetail implements Serializable {
   }
 
   private List<OrmQueryProperties> removeSecondaryQueries(boolean lazyQuery) {
-
     ArrayList<String> matchingPaths = new ArrayList<>(2);
-
     for (OrmQueryProperties chunk : fetchPaths.values()) {
       boolean match = lazyQuery ? chunk.isLazyFetch() : chunk.isQueryFetch();
       if (match) {
@@ -191,7 +180,6 @@ public class OrmQueryDetail implements Serializable {
 
     // sort into depth order to remove
     Collections.sort(matchingPaths);
-
     // the list of secondary queries
     ArrayList<OrmQueryProperties> props = new ArrayList<>();
 
@@ -225,14 +213,11 @@ public class OrmQueryDetail implements Serializable {
       OrmQueryProperties chunk = getChunk(split[0], true);
       chunk.addSecondaryQueryJoin(split[1]);
     }
-
     return props;
   }
 
   boolean tuneFetchProperties(OrmQueryDetail tunedDetail) {
-
     boolean tuned = false;
-
     OrmQueryProperties tunedRoot = tunedDetail.getChunk(null, false);
     if (tunedRoot != null) {
       tuned = true;
@@ -257,10 +242,7 @@ public class OrmQueryDetail implements Serializable {
   }
 
   /**
-   * Remove all joins and properties.
-   * <p>
-   * Typically for the row count query.
-   * </p>
+   * Remove all joins and properties. Typically for the row count query.
    */
   public void clear() {
     fetchPaths.clear();
@@ -306,7 +288,6 @@ public class OrmQueryDetail implements Serializable {
   }
 
   private void sortFetchPaths(BeanDescriptor<?> d, boolean addIds) {
-
     if (!fetchPaths.isEmpty()) {
       LinkedHashMap<String, OrmQueryProperties> sorted = new LinkedHashMap<>();
       for (OrmQueryProperties p : fetchPaths.values()) {
@@ -317,7 +298,6 @@ public class OrmQueryDetail implements Serializable {
   }
 
   private void sortFetchPaths(BeanDescriptor<?> d, OrmQueryProperties p, LinkedHashMap<String, OrmQueryProperties> sorted, boolean addId) {
-
     String path = p.getPath();
     if (!sorted.containsKey(path)) {
       String parentPath = p.getParentPath();
@@ -339,7 +319,6 @@ public class OrmQueryDetail implements Serializable {
             parentProp = new OrmQueryProperties(parentPath, Collections.emptySet());
           }
         }
-
         sortFetchPaths(d, parentProp, sorted, addId);
         sorted.put(path, p);
       }
@@ -350,14 +329,12 @@ public class OrmQueryDetail implements Serializable {
    * Mark 'fetch joins' to 'many' properties over to 'query joins' where needed.
    */
   void markQueryJoins(BeanDescriptor<?> beanDescriptor, String lazyLoadManyPath, boolean allowOne, boolean addIds) {
-
     if (fetchPaths.isEmpty()) {
       return;
     }
 
     // the name of the many fetch property if there is one
     String manyFetchProperty = null;
-
     // flag that is set once the many fetch property is chosen
     boolean fetchJoinFirstMany = allowOne;
 
@@ -375,6 +352,7 @@ public class OrmQueryDetail implements Serializable {
             // letting the first one remain a 'fetch join'
             fetchJoinFirstMany = false;
             manyFetchProperty = pair.getPath();
+            chunk.filterManyInline();
           } else {
             // convert this one over to a 'query join'
             chunk.markForQueryJoin();
@@ -388,7 +366,6 @@ public class OrmQueryDetail implements Serializable {
    * Sort the fetch entries taking into account fetchPreference on the path.
    */
   private List<FetchEntry> sortByFetchPreference(BeanDescriptor<?> desc) {
-
     List<FetchEntry> entries = new ArrayList<>(fetchPaths.size());
     int idx = 0;
     for (Map.Entry<String, OrmQueryProperties> entry : fetchPaths.entrySet()) {
@@ -452,11 +429,9 @@ public class OrmQueryDetail implements Serializable {
    * </p>
    */
   public void setDefaultSelectClause(BeanDescriptor<?> desc) {
-
     if (desc.hasDefaultSelectClause() && !hasSelectClause()) {
       baseProps = new OrmQueryProperties(null, desc.getDefaultSelectClause());
     }
-
     for (OrmQueryProperties joinProps : fetchPaths.values()) {
       if (!joinProps.hasSelectClause()) {
         BeanDescriptor<?> assocDesc = desc.getBeanDescriptor(joinProps.getPath());
@@ -506,7 +481,6 @@ public class OrmQueryDetail implements Serializable {
       props = new OrmQueryProperties(path);
       fetch(props);
       return props;
-
     } else {
       return props;
     }
@@ -516,9 +490,7 @@ public class OrmQueryDetail implements Serializable {
    * Return true if the fetch path is included.
    */
   public boolean includesPath(String path) {
-
     OrmQueryProperties chunk = fetchPaths.get(path);
-
     // may not have fetch properties if just +cache etc
     return chunk != null && !chunk.isCache();
   }
@@ -537,6 +509,15 @@ public class OrmQueryDetail implements Serializable {
     return fetchPaths.entrySet();
   }
 
+  /**
+   * Prepare filterMany expressions that are being included into the main query.
+   */
+  public void prepareExpressions(BeanQueryRequest<?> request) {
+    for (OrmQueryProperties value : fetchPaths.values()) {
+      value.prepareExpressions(request);
+    }
+  }
+
   private static class FetchEntry implements Comparable<FetchEntry> {
 
     private final int index;

ebean-core/src/main/java/io/ebeaninternal/server/querydefn/OrmQueryProperties.java

@@ -4,6 +4,7 @@ import io.ebean.ExpressionFactory;
 import io.ebean.FetchConfig;
 import io.ebean.OrderBy;
 import io.ebean.Query;
+import io.ebean.event.BeanQueryRequest;
 import io.ebean.util.SplitName;
 import io.ebeaninternal.api.SpiExpression;
 import io.ebeaninternal.api.SpiExpressionFactory;
@@ -178,8 +179,6 @@ public class OrmQueryProperties implements Serializable {
       SpiExpressionFactory queryEf = (SpiExpressionFactory) rootQuery.getExpressionFactory();
       ExpressionFactory filterEf = queryEf.createExpressionFactory();// exprPath);
       filterMany = new FilterExpressionList(exprPath, filterEf, rootQuery);
-      // by default we need to make this a 'query join' now
-      markForQueryJoin = true;
     }
     return filterMany;
   }
@@ -194,6 +193,24 @@ public class OrmQueryProperties implements Serializable {
     return filterMany.trimPath(trimPath);
   }
 
+  /**
+   * Adjust filterMany expressions for inclusion in main query.
+   */
+  public void filterManyInline() {
+    if (filterMany != null){
+      filterMany.prefixProperty(path);
+    }
+  }
+
+  /**
+   * Prepare filterMany expressions for query plan key.
+   */
+  public void prepareExpressions(BeanQueryRequest<?> request) {
+    if (filterMany != null) {
+      filterMany.prepareExpression(request);
+    }
+  }
+
   /**
    * Return the filterMany expression list (can be null).
    */
@@ -206,7 +223,6 @@ public class OrmQueryProperties implements Serializable {
    */
   public void setFilterMany(SpiExpressionList<?> filterMany) {
     this.filterMany = filterMany;
-    this.markForQueryJoin = true;
   }
 
   /**

ebean-core/src/main/java/io/ebeaninternal/server/transaction/DefaultPersistenceContext.java

@@ -2,6 +2,9 @@ package io.ebeaninternal.server.transaction;
 
 import io.ebean.bean.EntityBean;
 import io.ebean.bean.PersistenceContext;
+import io.ebeaninternal.api.SpiBeanType;
+import io.ebeaninternal.api.SpiBeanTypeManager;
+import io.ebeaninternal.api.SpiPersistenceContext;
 
 import java.util.*;
 import java.util.concurrent.locks.ReentrantLock;
@@ -11,19 +14,16 @@ import java.util.concurrent.locks.ReentrantLock;
  * <p>
  * Ensures only one instance of a bean is used according to its type and unique
  * id.
- * </p>
  * <p>
  * PersistenceContext lives on a Transaction and as such is expected to only
  * have a single thread accessing it at a time. This is not expected to be used
  * concurrently.
- * </p>
  * <p>
  * Duplicate beans are ones having the same type and unique id value. These are
  * considered duplicates and replaced by the bean instance that was already
  * loaded into the PersistenceContext.
- * </p>
  */
-public final class DefaultPersistenceContext implements PersistenceContext {
+public final class DefaultPersistenceContext implements SpiPersistenceContext {
 
   /**
    * Map used hold caches. One cache per bean type.
@@ -199,12 +199,12 @@ public final class DefaultPersistenceContext implements PersistenceContext {
   }
 
   @Override
-  public List<Object> dirtyBeans() {
+  public List<Object> dirtyBeans(SpiBeanTypeManager manager) {
     lock.lock();
     try {
       List<Object> list = new ArrayList<>();
       for (ClassContext classContext : typeCache.values()) {
-        classContext.dirtyBeans(list);
+        classContext.dirtyBeans(manager, list);
       }
       return list;
     } finally {
@@ -223,26 +223,26 @@ public final class DefaultPersistenceContext implements PersistenceContext {
   }
 
   private ClassContext getClassContext(Class<?> rootType) {
-    return typeCache.computeIfAbsent(rootType, k -> new ClassContext());
+    return typeCache.computeIfAbsent(rootType, k -> new ClassContext(rootType));
   }
 
   private static class ClassContext {
 
     private final Map<Object, Object> map = new HashMap<>();
-
+    private final Class<?> rootType;
     private Set<Object> deleteSet;
-
     private int initialSize;
-
     private ClassContext parent;
 
-    private ClassContext() {
+    private ClassContext(Class<?> rootType) {
+      this.rootType = rootType;
     }
 
     /**
      * Create as a shallow copy.
      */
     private ClassContext(ClassContext source, boolean initial) {
+      this.rootType = source.rootType;
       if (initial || source.isTransfer()) {
         parent = source.transferParent();
         initialSize = parent.size();
@@ -334,9 +334,11 @@ public final class DefaultPersistenceContext implements PersistenceContext {
     /**
      * Add the dirty beans to the list.
      */
-    void dirtyBeans(List<Object> list) {
+    void dirtyBeans(SpiBeanTypeManager manager, List<Object> list) {
+      final SpiBeanType beanType = manager.getBeanType(rootType);
       for (Object value : map.values()) {
-        if (((EntityBean) value)._ebean_getIntercept().isDirty()) {
+        EntityBean bean = (EntityBean) value;
+        if (bean._ebean_getIntercept().isDirty() || beanType.isToManyDirty(bean)) {
           list.add(value);
         }
       }

ebean-core/src/main/java/io/ebeaninternal/server/transaction/ImplicitReadOnlyTransaction.java

@@ -3,13 +3,9 @@ package io.ebeaninternal.server.transaction;
 import io.ebean.ProfileLocation;
 import io.ebean.TransactionCallback;
 import io.ebean.annotation.DocStoreMode;
-import io.ebean.bean.PersistenceContext;
 import io.ebean.event.changelog.BeanChange;
 import io.ebean.event.changelog.ChangeSet;
-import io.ebeaninternal.api.SpiProfileTransactionEvent;
-import io.ebeaninternal.api.SpiTransaction;
-import io.ebeaninternal.api.TransactionEvent;
-import io.ebeaninternal.api.TxnProfileEventCodes;
+import io.ebeaninternal.api.*;
 import io.ebeaninternal.server.core.PersistDeferredRelationship;
 import io.ebeaninternal.server.core.PersistRequestBean;
 import io.ebeaninternal.server.persist.BatchControl;
@@ -58,7 +54,7 @@ class ImplicitReadOnlyTransaction implements SpiTransaction, TxnProfileEventCode
   /**
    * Holder of the objects fetched to ensure unique objects are used.
    */
-  private PersistenceContext persistenceContext;
+  private SpiPersistenceContext persistenceContext;
 
   private Object tenantId;
 
@@ -421,7 +417,7 @@ class ImplicitReadOnlyTransaction implements SpiTransaction, TxnProfileEventCode
    * Return the persistence context associated with this transaction.
    */
   @Override
-  public PersistenceContext getPersistenceContext() {
+  public SpiPersistenceContext getPersistenceContext() {
     return persistenceContext;
   }
 
@@ -434,7 +430,7 @@ class ImplicitReadOnlyTransaction implements SpiTransaction, TxnProfileEventCode
    * </p>
    */
   @Override
-  public void setPersistenceContext(PersistenceContext context) {
+  public void setPersistenceContext(SpiPersistenceContext context) {
     if (!isActive()) {
       throw new IllegalStateException(illegalStateMessage);
     }

ebean-core/src/main/java/io/ebeaninternal/server/transaction/JdbcTransaction.java

@@ -3,20 +3,16 @@ package io.ebeaninternal.server.transaction;
 import io.ebean.ProfileLocation;
 import io.ebean.TransactionCallback;
 import io.ebean.annotation.DocStoreMode;
-import io.ebean.bean.PersistenceContext;
 import io.ebean.config.DatabaseConfig;
 import io.ebean.config.dbplatform.DatabasePlatform.OnQueryOnly;
 import io.ebean.event.changelog.BeanChange;
 import io.ebean.event.changelog.ChangeSet;
-import io.ebeaninternal.api.SpiProfileTransactionEvent;
-import io.ebeaninternal.api.SpiTransaction;
-import io.ebeaninternal.api.TransactionEvent;
-import io.ebeaninternal.api.TxnProfileEventCodes;
+import io.ebeaninternal.api.*;
 import io.ebeaninternal.server.core.PersistDeferredRelationship;
 import io.ebeaninternal.server.core.PersistRequestBean;
-import io.ebeaninternal.server.lib.Str;
 import io.ebeaninternal.server.persist.BatchControl;
 import io.ebeaninternal.server.persist.BatchedSqlException;
+import io.ebeaninternal.server.util.Str;
 import io.ebeanservice.docstore.api.DocStoreTransaction;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -25,12 +21,7 @@ import javax.persistence.PersistenceException;
 import javax.persistence.RollbackException;
 import java.sql.Connection;
 import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.IdentityHashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.function.Consumer;
 
 /**
@@ -99,7 +90,7 @@ class JdbcTransaction implements SpiTransaction, TxnProfileEventCodes {
   /**
    * Holder of the objects fetched to ensure unique objects are used.
    */
-  private PersistenceContext persistenceContext;
+  private SpiPersistenceContext persistenceContext;
 
   /**
    * Used to give developers more control over the insert update and delete
@@ -806,7 +797,7 @@ class JdbcTransaction implements SpiTransaction, TxnProfileEventCodes {
    * Return the persistence context associated with this transaction.
    */
   @Override
-  public PersistenceContext getPersistenceContext() {
+  public SpiPersistenceContext getPersistenceContext() {
     return persistenceContext;
   }
 
@@ -816,10 +807,9 @@ class JdbcTransaction implements SpiTransaction, TxnProfileEventCodes {
    * This could be considered similar to EJB3 Extended PersistanceContext. In
    * that you get the PersistanceContext from a transaction, hold onto it, and
    * then set it back later to a second transaction.
-   * </p>
    */
   @Override
-  public void setPersistenceContext(PersistenceContext context) {
+  public void setPersistenceContext(SpiPersistenceContext context) {
     if (!isActive()) {
       throw new IllegalStateException(illegalStateMessage);
     }
@@ -951,11 +941,13 @@ class JdbcTransaction implements SpiTransaction, TxnProfileEventCodes {
    */
   private void connectionEndForQueryOnly() {
     try {
+      withEachCallback(TransactionCallback::preCommit);
       if (onQueryOnly == OnQueryOnly.COMMIT) {
         performCommit();
       } else {
         performRollback();
       }
+      withEachCallback(TransactionCallback::postCommit);
     } catch (SQLException e) {
       logger.error("Error when ending a query only transaction via " + onQueryOnly, e);
     }

ebean-core/src/main/java/io/ebeaninternal/server/transaction/NoTransaction.java

@@ -3,9 +3,9 @@ package io.ebeaninternal.server.transaction;
 import io.ebean.ProfileLocation;
 import io.ebean.TransactionCallback;
 import io.ebean.annotation.DocStoreMode;
-import io.ebean.bean.PersistenceContext;
 import io.ebean.event.changelog.BeanChange;
 import io.ebean.event.changelog.ChangeSet;
+import io.ebeaninternal.api.SpiPersistenceContext;
 import io.ebeaninternal.api.SpiProfileTransactionEvent;
 import io.ebeaninternal.api.SpiTransaction;
 import io.ebeaninternal.api.TransactionEvent;
@@ -349,12 +349,12 @@ class NoTransaction implements SpiTransaction {
   }
 
   @Override
-  public PersistenceContext getPersistenceContext() {
+  public SpiPersistenceContext getPersistenceContext() {
     return null;
   }
 
   @Override
-  public void setPersistenceContext(PersistenceContext context) {
+  public void setPersistenceContext(SpiPersistenceContext context) {
   }
 
   @Override

ebean-core/src/main/java/io/ebeaninternal/server/transaction/SavepointTransaction.java

@@ -2,7 +2,7 @@ package io.ebeaninternal.server.transaction;
 
 import io.ebeaninternal.api.SpiTransaction;
 import io.ebeaninternal.api.SpiTransactionProxy;
-import io.ebeaninternal.server.lib.Str;
+import io.ebeaninternal.server.util.Str;
 
 import javax.persistence.PersistenceException;
 import java.sql.Connection;

ebean-core/src/main/java/io/ebeaninternal/server/transaction/TransactionManager.java

@@ -5,7 +5,6 @@ import io.ebean.ProfileLocation;
 import io.ebean.TxScope;
 import io.ebean.annotation.PersistBatch;
 import io.ebean.annotation.TxType;
-import io.ebean.bean.PersistenceContext;
 import io.ebean.cache.ServerCacheNotification;
 import io.ebean.cache.ServerCacheNotify;
 import io.ebean.config.CurrentTenantProvider;
@@ -19,15 +18,7 @@ import io.ebean.metric.MetricFactory;
 import io.ebean.metric.TimedMetric;
 import io.ebean.metric.TimedMetricMap;
 import io.ebean.plugin.SpiServer;
-import io.ebeaninternal.api.ScopeTrans;
-import io.ebeaninternal.api.ScopedTransaction;
-import io.ebeaninternal.api.SpiLogManager;
-import io.ebeaninternal.api.SpiLogger;
-import io.ebeaninternal.api.SpiProfileHandler;
-import io.ebeaninternal.api.SpiTransaction;
-import io.ebeaninternal.api.SpiTransactionManager;
-import io.ebeaninternal.api.TransactionEvent;
-import io.ebeaninternal.api.TransactionEventTable;
+import io.ebeaninternal.api.*;
 import io.ebeaninternal.api.TransactionEventTable.TableIUD;
 import io.ebeaninternal.server.cache.CacheChangeSet;
 import io.ebeaninternal.server.cluster.ClusterManager;
@@ -53,7 +44,6 @@ import java.util.concurrent.atomic.AtomicLong;
  * Manages transactions.
  * <p>
  * Keeps the Cache and Cluster in sync when transactions are committed.
- * </p>
  */
 public class TransactionManager implements SpiTransactionManager {
 
@@ -198,8 +188,6 @@ public class TransactionManager implements SpiTransactionManager {
     this.txnMain = metricFactory.createTimedMetric("txn.main");
     this.txnReadOnly = metricFactory.createTimedMetric("txn.readonly");
     this.txnNamed = metricFactory.createTimedMetricMap("txn.named.");
-    // Add gauges for db pool size
-
     scopeManager.register(this);
   }
 
@@ -304,20 +292,16 @@ public class TransactionManager implements SpiTransactionManager {
    * There is a potential optimisation available when read committed is the default
    * isolation level. If it is, then Connections used only for queries do not require
    * commit or rollback but instead can just be put back into the pool via close().
-   * </p>
    * <p>
    * If the Isolation level is higher (say SERIALIZABLE) then Connections used
    * just for queries do need to be committed or rollback after the query.
-   * </p>
    */
   OnQueryOnly initOnQueryOnly(OnQueryOnly dbPlatformOnQueryOnly) {
-
     // first check for a system property 'override'
     String systemPropertyValue = System.getProperty("ebean.transaction.onqueryonly");
     if (systemPropertyValue != null) {
       return OnQueryOnly.valueOf(systemPropertyValue.trim().toUpperCase());
     }
-
     // default to rollback if not defined on the platform
     return dbPlatformOnQueryOnly == null ? OnQueryOnly.COMMIT : dbPlatformOnQueryOnly;
   }
@@ -359,9 +343,7 @@ public class TransactionManager implements SpiTransactionManager {
    * Wrap an externally supplied Connection with a known transaction id.
    */
   private SpiTransaction wrapExternalConnection(String id, Connection c) {
-
     ExternalJdbcTransaction t = new ExternalJdbcTransaction(id, true, c, this);
-
     // set the default batch mode
     t.setBatchMode(persistBatch);
     t.setBatchOnCascade(persistBatchOnCascade);
@@ -409,7 +391,6 @@ public class TransactionManager implements SpiTransactionManager {
    */
   @Override
   public void notifyOfRollback(SpiTransaction transaction, Throwable cause) {
-
     try {
       if (txnLogger.isDebug()) {
         String msg = transaction.getLogPrefix() + "Rollback";
@@ -418,7 +399,6 @@ public class TransactionManager implements SpiTransactionManager {
         }
         txnLogger.debug(msg);
       }
-
     } catch (Exception ex) {
       logger.error("Error while notifying TransactionEventListener of rollback event", ex);
     }
@@ -445,7 +425,6 @@ public class TransactionManager implements SpiTransactionManager {
   }
 
   private void formatThrowable(Throwable e, StringBuilder sb) {
-
     sb.append(e.toString());
     StackTraceElement[] stackTrace = e.getStackTrace();
     if (stackTrace.length > 0) {
@@ -468,11 +447,9 @@ public class TransactionManager implements SpiTransactionManager {
       if (txnLogger.isDebug()) {
         txnLogger.debug(transaction.getLogPrefix() + "Commit");
       }
-
       PostCommitProcessing postCommit = new PostCommitProcessing(clusterManager, this, transaction);
       postCommit.notifyLocalCache();
       backgroundExecutor.execute(postCommit.backgroundNotify());
-
     } catch (Exception ex) {
       logger.error("NotifyOfCommit failed. L2 Cache potentially not notified.", ex);
     }
@@ -488,7 +465,6 @@ public class TransactionManager implements SpiTransactionManager {
   }
 
   private void externalModificationEvent(TransactionEventTable tableEvents) {
-
     TransactionEvent event = new TransactionEvent();
     event.add(tableEvents);
 
@@ -501,25 +477,21 @@ public class TransactionManager implements SpiTransactionManager {
    * Notify local BeanPersistListeners etc of events from another server in the cluster.
    */
   public void remoteTransactionEvent(RemoteTransactionEvent remoteEvent) {
-
     if (clusterLogger.isDebugEnabled()) {
       clusterLogger.debug("processing {}", remoteEvent);
     }
-
     CacheChangeSet changeSet = new CacheChangeSet();
 
     RemoteTableMod tableMod = remoteEvent.getRemoteTableMod();
     if (tableMod != null) {
       changeSet.addInvalidate(tableMod.getTables());
     }
-
     List<TableIUD> tableIUDList = remoteEvent.getTableIUDList();
     if (tableIUDList != null) {
       for (TableIUD tableIUD : tableIUDList) {
         beanDescriptorManager.cacheNotify(tableIUD, changeSet);
       }
     }
-
     // note DeleteById is written as BeanPersistIds and getBeanPersistList()
     // processes both Bean IUD and DeleteById
     List<BeanPersistIds> beanPersistList = remoteEvent.getBeanPersistList();
@@ -528,7 +500,6 @@ public class TransactionManager implements SpiTransactionManager {
         persistIds.notifyCache(changeSet);
       }
     }
-
     changeSet.apply();
   }
 
@@ -543,10 +514,8 @@ public class TransactionManager implements SpiTransactionManager {
    * Prepare and then send/log the changeSet.
    */
   void sendChangeLog(final ChangeSet changeSet) {
-
     // can set userId, userIpAddress & userContext if desired
     if (changeLogPrepare.prepare(changeSet)) {
-
       if (changeLogAsync) {
         // call the log method in background
         backgroundExecutor.execute(() -> changeLogListener.log(changeSet));
@@ -653,9 +622,7 @@ public class TransactionManager implements SpiTransactionManager {
    * Begin a scoped transaction.
    */
   public ScopedTransaction beginScopedTransaction(TxScope txScope) {
-
     txScope = initTxScope(txScope);
-
     ScopedTransaction txnContainer = getActiveScoped();
 
     boolean setToScope;
@@ -678,7 +645,6 @@ public class TransactionManager implements SpiTransactionManager {
     if (nestedSavepoint && (type == TxType.REQUIRED || type == TxType.REQUIRES_NEW)) {
       createTransaction = true;
       transaction = createSavepoint(transaction, this);
-
     } else {
       createTransaction = isCreateNewTransaction(transaction, type);
       if (createTransaction) {
@@ -748,32 +714,26 @@ public class TransactionManager implements SpiTransactionManager {
    * Determine whether to create a new transaction or not.
    * <p>
    * This will also potentially throw exceptions for MANDATORY and NEVER types.
-   * </p>
    */
   private boolean isCreateNewTransaction(SpiTransaction current, TxType type) {
     switch (type) {
       case REQUIRED:
       case SUPPORTS:
         return current == null;
-
       case REQUIRES_NEW:
         return true;
-
       case MANDATORY:
         if (current == null) {
           throw new PersistenceException("Transaction missing when MANDATORY");
         }
         return false;
-
       case NEVER:
         if (current != null) {
           throw new PersistenceException("Transaction exists for Transactional NEVER");
         }
         return true; // always use NoTransaction instance
-
       case NOT_SUPPORTED:
         return true; // always use NoTransaction instance
-
       default:
         throw new RuntimeException("Should never get here?");
     }
@@ -801,8 +761,8 @@ public class TransactionManager implements SpiTransactionManager {
   /**
    * Experimental - find dirty beans in the persistence context and persist them.
    */
-  public void flushTransparent(PersistenceContext persistenceContext, SpiTransaction transaction) {
-    List<Object> dirtyBeans = persistenceContext.dirtyBeans();
+  public void flushTransparent(SpiPersistenceContext persistenceContext, SpiTransaction transaction) {
+    List<Object> dirtyBeans = persistenceContext.dirtyBeans(beanDescriptorManager);
     if (!dirtyBeans.isEmpty()) {
       server.updateAll(dirtyBeans, transaction);
     }

ebean-core/src/main/java/io/ebeaninternal/server/type/CheckMarkedDirty.java

@@ -1,26 +0,0 @@
-package io.ebeaninternal.server.type;
-
-import io.ebean.ModifyAwareType;
-
-/**
- * Check dirty state of json value which might be modify aware.
- */
-class CheckMarkedDirty {
-
-  /**
-   * Return true if the value should be considered dirty (and included in an update).
-   */
-  static boolean isDirty(Object value) {
-    if (value instanceof ModifyAwareType) {
-      ModifyAwareType modifyAware = (ModifyAwareType) value;
-      if (modifyAware.isMarkedDirty()) {
-        // reset the dirty state (consider not dirty after update)
-        modifyAware.setMarkedDirty(false);
-        return true;
-      } else {
-        return false;
-      }
-    }
-    return true;
-  }
-}

ebean-core/src/main/java/io/ebeaninternal/server/type/DefaultTypeManager.java

@@ -3,11 +3,7 @@ package io.ebeaninternal.server.type;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.introspect.AnnotatedField;
-import io.ebean.annotation.DbArray;
-import io.ebean.annotation.DbEnumType;
-import io.ebean.annotation.DbEnumValue;
-import io.ebean.annotation.EnumValue;
-import io.ebean.annotation.Platform;
+import io.ebean.annotation.*;
 import io.ebean.config.DatabaseConfig;
 import io.ebean.config.JsonConfig;
 import io.ebean.config.PlatformConfig;
@@ -34,125 +30,70 @@ import org.slf4j.LoggerFactory;
 import javax.persistence.AttributeConverter;
 import javax.persistence.EnumType;
 import java.io.File;
-import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
 import java.lang.reflect.ParameterizedType;
 import java.lang.reflect.Type;
 import java.math.BigDecimal;
 import java.math.BigInteger;
-import java.net.Inet4Address;
-import java.net.Inet6Address;
-import java.net.InetAddress;
-import java.net.URI;
-import java.net.URL;
+import java.net.*;
 import java.sql.Date;
 import java.sql.Time;
 import java.sql.Timestamp;
 import java.sql.Types;
-import java.time.DayOfWeek;
-import java.time.Duration;
-import java.time.Instant;
-import java.time.Month;
-import java.time.MonthDay;
-import java.time.OffsetDateTime;
-import java.time.OffsetTime;
-import java.time.Year;
-import java.time.YearMonth;
-import java.time.ZoneId;
-import java.time.ZoneOffset;
-import java.time.ZonedDateTime;
-import java.util.Arrays;
-import java.util.Calendar;
-import java.util.Currency;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.ServiceLoader;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.UUID;
+import java.time.*;
+import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * Default implementation of TypeManager.
  * <p>
  * Manages the list of ScalarType that is available.
- * </p>
  */
 public final class DefaultTypeManager implements TypeManager {
 
   private static final Logger logger = LoggerFactory.getLogger(DefaultTypeManager.class);
 
   private final ConcurrentHashMap<Class<?>, ScalarType<?>> typeMap;
-
   private final ConcurrentHashMap<Integer, ScalarType<?>> nativeMap;
-
   private final ConcurrentHashMap<String, ScalarType<?>> logicalMap;
 
   private final DefaultTypeFactory extraTypeFactory;
 
   private final ScalarType<?> hstoreType = new ScalarTypePostgresHstore();
-
   private final ScalarTypeFile fileType = new ScalarTypeFile();
-
   private final ScalarType<?> charType = new ScalarTypeChar();
-
   private final ScalarType<?> charArrayType = new ScalarTypeCharArray();
-
   private final ScalarType<?> longVarcharType = new ScalarTypeLongVarchar();
-
   private final ScalarType<?> clobType = new ScalarTypeClob();
-
   private final ScalarType<?> byteType = new ScalarTypeByte();
-
   private final ScalarType<?> binaryType = new ScalarTypeBytesBinary();
-
   private final ScalarType<?> blobType = new ScalarTypeBytesBlob();
-
   private final ScalarType<?> varbinaryType = new ScalarTypeBytesVarbinary();
-
   private final ScalarType<?> longVarbinaryType = new ScalarTypeBytesLongVarbinary();
-
   private final ScalarType<?> shortType = new ScalarTypeShort();
-
   private final ScalarType<?> integerType = ScalarTypeInteger.INSTANCE;
-
   private final ScalarType<?> longType = new ScalarTypeLong();
-
   private final ScalarType<?> doubleType = new ScalarTypeDouble();
-
   private final ScalarType<?> floatType = new ScalarTypeFloat();
-
   private final ScalarType<?> bigDecimalType = new ScalarTypeBigDecimal();
-
   private final ScalarType<?> timeType = new ScalarTypeTime();
-
   private final ScalarType<?> urlType = new ScalarTypeURL();
   private final ScalarType<?> uriType = new ScalarTypeURI();
   private final ScalarType<?> localeType = new ScalarTypeLocale();
   private final ScalarType<?> currencyType = new ScalarTypeCurrency();
   private final ScalarType<?> timeZoneType = new ScalarTypeTimeZone();
-
   private final ScalarType<?> stringType = ScalarTypeString.INSTANCE;
-
   private final ScalarType<?> classType = new ScalarTypeClass();
 
   private final JsonConfig.DateTime jsonDateTime;
   private final JsonConfig.Date jsonDate;
 
   private final Object objectMapper;
-
   private final boolean objectMapperPresent;
-
   private final boolean postgres;
-
+  private final TypeJsonManager jsonManager;
   private final boolean offlineMigrationGeneration;
-
   private final EnumType defaultEnumType;
 
   // OPTIONAL ScalarTypes registered if Jackson/JsonNode is in the classpath
@@ -191,24 +132,20 @@ public final class DefaultTypeManager implements TypeManager {
     this.typeMap = new ConcurrentHashMap<>();
     this.nativeMap = new ConcurrentHashMap<>();
     this.logicalMap = new ConcurrentHashMap<>();
-
+    this.postgres = isPostgres(config.getDatabasePlatform());
     this.objectMapperPresent = config.getClassLoadConfig().isJacksonObjectMapperPresent();
     this.objectMapper = (objectMapperPresent) ? initObjectMapper(config) : null;
-
+    this.jsonManager = (objectMapperPresent) ? new TypeJsonManager(postgres, objectMapper, config.isJsonDirtyByDefault()) : null;
     this.extraTypeFactory = new DefaultTypeFactory(config);
-    this.postgres = isPostgres(config.getDatabasePlatform());
     this.arrayTypeListFactory = arrayTypeListFactory(config.getDatabasePlatform());
     this.arrayTypeSetFactory = arrayTypeSetFactory(config.getDatabasePlatform());
-
     this.offlineMigrationGeneration = DbOffline.isGenerateMigration();
-
     this.defaultEnumType = config.getDefaultEnumType();
 
     initialiseStandard(config);
     initialiseJavaTimeTypes(config);
     initialiseJodaTypes(config);
     initialiseJacksonTypes(config);
-
     loadTypesFromProviders(config, objectMapper);
     loadGeoTypeBinder(config);
 
@@ -489,7 +426,7 @@ public final class DefaultTypeManager implements TypeManager {
     if (objectMapper == null) {
       throw new IllegalArgumentException("Type [" + type + "] unsupported for @DbJson mapping - Jackson ObjectMapper not present");
     }
-    return ScalarTypeJsonObjectMapper.createTypeFor(postgres, (AnnotatedField) prop.getJacksonField(), (ObjectMapper) objectMapper, dbType, docType);
+    return ScalarTypeJsonObjectMapper.createTypeFor(jsonManager, (AnnotatedField) prop.getJacksonField(), dbType, docType);
   }
 
   /**
@@ -588,8 +525,7 @@ public final class DefaultTypeManager implements TypeManager {
   private ScalarTypeEnum<?> createEnumScalarType2(Class<?> enumType) {
     boolean integerType = true;
     Map<String, String> nameValueMap = new LinkedHashMap<>();
-    Field[] fields = enumType.getDeclaredFields();
-    for (Field field : fields) {
+    for (Field field : enumType.getDeclaredFields()) {
       EnumValue enumValue = AnnotationUtil.get(field, EnumValue.class);
       if (enumValue != null) {
         nameValueMap.put(field.getName(), enumValue.value());
@@ -652,8 +588,7 @@ public final class DefaultTypeManager implements TypeManager {
   }
 
   private ScalarTypeEnum<?> createEnumScalarTypePerExtentions(Class<? extends Enum<?>> enumType) {
-    Method[] methods = enumType.getMethods();
-    for (Method method : methods) {
+    for (Method method : enumType.getMethods()) {
       DbEnumValue dbValue = AnnotationUtil.get(method, DbEnumValue.class);
       if (dbValue != null) {
         boolean integerValues = DbEnumType.INTEGER == dbValue.storage();
@@ -668,7 +603,6 @@ public final class DefaultTypeManager implements TypeManager {
    * Create the Mapping of Enum fields to DB values using EnumValue annotations.
    * <p>
    * Return null if the EnumValue annotations are not present/used.
-   * </p>
    */
   private ScalarTypeEnum<?> createEnumScalarTypeDbValue(Class<? extends Enum<?>> enumType, Method method, boolean integerType, int length, boolean withConstraint) {
     Map<String, String> nameValueMap = new LinkedHashMap<>();
@@ -697,11 +631,11 @@ public final class DefaultTypeManager implements TypeManager {
     EnumToDbValueMap<?> beanDbMap = EnumToDbValueMap.create(integerType);
     int maxValueLen = 0;
     for (Map.Entry<String, String> entry : nameValueMap.entrySet()) {
-      String name = entry.getKey();
+      String name = entry.getKey().trim();
       String value = entry.getValue();
       maxValueLen = Math.max(maxValueLen, value.length());
-      Object enumValue = Enum.valueOf(enumType, name.trim());
-      beanDbMap.add(enumValue, value, name.trim());
+      Object enumValue = Enum.valueOf(enumType, name);
+      beanDbMap.add(enumValue, value, name);
     }
     if (dbColumnLength == 0 && !integerType) {
       dbColumnLength = maxValueLen;
@@ -711,31 +645,24 @@ public final class DefaultTypeManager implements TypeManager {
 
   /**
    * Automatically find any ScalarTypes by searching through the class path.
-   * <p>
-   * In avaje.properties define a list of packages in which ScalarTypes are
-   * found. This will search for any class that implements the ScalarType
-   * interface and register it with this TypeManager.
-   * </p>
    */
   private void initialiseCustomScalarTypes(BootupClasses bootupClasses) {
     for (Class<? extends ScalarType<?>> cls : bootupClasses.getScalarTypes()) {
       try {
         ScalarType<?> scalarType;
         if (objectMapper == null) {
-          scalarType = cls.newInstance();
+          scalarType = cls.getDeclaredConstructor().newInstance();
         } else {
           try {
             // first try objectMapper constructor
-            Constructor<? extends ScalarType<?>> constructor = cls.getConstructor(ObjectMapper.class);
-            scalarType = constructor.newInstance((ObjectMapper) objectMapper);
+            scalarType = cls.getDeclaredConstructor(ObjectMapper.class).newInstance(objectMapper);
           } catch (NoSuchMethodException e) {
-            scalarType = cls.newInstance();
+            scalarType = cls.getDeclaredConstructor().newInstance();
           }
         }
         addCustomType(scalarType);
       } catch (Exception e) {
-        String msg = "Error loading ScalarType [" + cls.getName() + "]";
-        logger.error(msg, e);
+        logger.error("Error loading ScalarType [" + cls.getName() + "]", e);
       }
     }
   }
@@ -755,8 +682,7 @@ public final class DefaultTypeManager implements TypeManager {
 
   @SuppressWarnings({"unchecked", "rawtypes"})
   private void initialiseScalarConverters(BootupClasses bootupClasses) {
-    List<Class<? extends ScalarTypeConverter<?, ?>>> foundTypes = bootupClasses.getScalarConverters();
-    for (Class<? extends ScalarTypeConverter<?, ?>> foundType : foundTypes) {
+    for (Class<? extends ScalarTypeConverter<?, ?>> foundType : bootupClasses.getScalarConverters()) {
       try {
         Class<?>[] paramTypes = TypeReflectHelper.getParams(foundType, ScalarTypeConverter.class);
         if (paramTypes.length != 2) {
@@ -768,7 +694,7 @@ public final class DefaultTypeManager implements TypeManager {
         if (wrappedType == null) {
           throw new IllegalStateException("Could not find ScalarType for: " + paramTypes[1]);
         }
-        ScalarTypeConverter converter = foundType.newInstance();
+        ScalarTypeConverter converter = foundType.getDeclaredConstructor().newInstance();
         ScalarTypeWrapper stw = new ScalarTypeWrapper(logicalType, wrappedType, converter);
         logger.debug("Register ScalarTypeWrapper from {} -> {} using:{}", logicalType, persistType, foundType);
         add(stw);
@@ -780,8 +706,7 @@ public final class DefaultTypeManager implements TypeManager {
 
   @SuppressWarnings({"unchecked", "rawtypes"})
   private void initialiseAttributeConverters(BootupClasses bootupClasses) {
-    List<Class<? extends AttributeConverter<?, ?>>> foundTypes = bootupClasses.getAttributeConverters();
-    for (Class<? extends AttributeConverter<?, ?>> foundType : foundTypes) {
+    for (Class<? extends AttributeConverter<?, ?>> foundType : bootupClasses.getAttributeConverters()) {
       try {
         Class<?>[] paramTypes = TypeReflectHelper.getParams(foundType, AttributeConverter.class);
         if (paramTypes.length != 2) {
@@ -793,7 +718,7 @@ public final class DefaultTypeManager implements TypeManager {
         if (wrappedType == null) {
           throw new IllegalStateException("Could not find ScalarType for: " + paramTypes[1]);
         }
-        AttributeConverter converter = foundType.newInstance();
+        AttributeConverter converter = foundType.getDeclaredConstructor().newInstance();
         ScalarTypeWrapper stw = new ScalarTypeWrapper(logicalType, wrappedType, new AttributeConverterAdapter(converter));
         logger.debug("Register ScalarTypeWrapper from {} -> {} using:{}", logicalType, persistType, foundType);
         add(stw);
@@ -825,12 +750,15 @@ public final class DefaultTypeManager implements TypeManager {
   }
 
   private void initialiseJavaTimeTypes(DatabaseConfig config) {
+
+    ZoneId zoneId = getZoneId(config);
+
     typeMap.put(java.nio.file.Path.class, new ScalarTypePath());
     addType(java.time.Period.class, new ScalarTypePeriod());
     addType(java.time.LocalDate.class, new ScalarTypeLocalDate(jsonDate));
     addType(java.time.LocalDateTime.class, new ScalarTypeLocalDateTime(jsonDateTime));
-    addType(OffsetDateTime.class, new ScalarTypeOffsetDateTime(jsonDateTime));
-    addType(ZonedDateTime.class, new ScalarTypeZonedDateTime(jsonDateTime));
+    addType(OffsetDateTime.class, new ScalarTypeOffsetDateTime(jsonDateTime, zoneId));
+    addType(ZonedDateTime.class, new ScalarTypeZonedDateTime(jsonDateTime, zoneId));
     addType(Instant.class, new ScalarTypeInstant(jsonDateTime));
     addType(DayOfWeek.class, new ScalarTypeDayOfWeek());
     addType(Month.class, new ScalarTypeMonth());
@@ -846,6 +774,11 @@ public final class DefaultTypeManager implements TypeManager {
     addType(Duration.class, (durationNanos) ? new ScalarTypeDurationWithNanos() : new ScalarTypeDuration());
   }
 
+  private ZoneId getZoneId(DatabaseConfig config) {
+    final String dataTimeZone = config.getDataTimeZone();
+    return (dataTimeZone == null) ? ZoneOffset.systemDefault() : TimeZone.getTimeZone(dataTimeZone).toZoneId();
+  }
+
   private void addType(Class<?> clazz, ScalarType<?> scalarType) {
     typeMap.put(clazz, scalarType);
     logicalMap.putIfAbsent(clazz.getSimpleName(), scalarType);
@@ -890,23 +823,16 @@ public final class DefaultTypeManager implements TypeManager {
 
     nativeMap.put(DbPlatformType.HSTORE, hstoreType);
 
-    ScalarType<?> utilDateType = extraTypeFactory.createUtilDate(jsonDateTime, jsonDate);
-    addType(java.util.Date.class, utilDateType);
-
-    ScalarType<?> calType = extraTypeFactory.createCalendar(jsonDateTime);
-    addType(Calendar.class, calType);
-
-    ScalarType<?> mathBigIntType = extraTypeFactory.createMathBigInteger();
-    addType(BigInteger.class, mathBigIntType);
+    addType(java.util.Date.class, extraTypeFactory.createUtilDate(jsonDateTime, jsonDate));
+    addType(Calendar.class, extraTypeFactory.createCalendar(jsonDateTime));
+    addType(BigInteger.class, extraTypeFactory.createMathBigInteger());
 
     ScalarTypeBool booleanType = extraTypeFactory.createBoolean();
     addType(Boolean.class, booleanType);
     addType(boolean.class, booleanType);
-
     // register the boolean literals to the platform for DDL default values
     databasePlatform.setDbTrueLiteral(booleanType.getDbTrueLiteral());
     databasePlatform.setDbFalseLiteral(booleanType.getDbFalseLiteral());
-
     // always register Types.BOOLEAN to our boolean type
     nativeMap.put(Types.BOOLEAN, booleanType);
     if (booleanType.getJdbcType() == Types.BIT) {
@@ -915,7 +841,6 @@ public final class DefaultTypeManager implements TypeManager {
     }
 
     PlatformConfig.DbUuid dbUuid = config.getPlatformConfig().getDbUuid();
-
     if (offlineMigrationGeneration || (databasePlatform.isNativeUuidType() && dbUuid.useNativeType())) {
       addType(UUID.class, new ScalarTypeUUIDNative());
     } else {
@@ -1027,6 +952,7 @@ public final class DefaultTypeManager implements TypeManager {
     nativeMap.put(Types.TIMESTAMP, timestampType);
   }
 
+  @SuppressWarnings("rawtypes")
   private void addInetAddressType(ScalarType scalarType) {
     addType(InetAddress.class, scalarType);
     addType(Inet4Address.class, scalarType);

ebean-core/src/main/java/io/ebeaninternal/server/type/JacksonTypeHelper.java

@@ -0,0 +1,46 @@
+package io.ebeaninternal.server.type;
+
+import com.fasterxml.jackson.databind.*;
+import com.fasterxml.jackson.databind.introspect.AnnotatedField;
+
+class JacksonTypeHelper {
+
+  private final AnnotatedField field;
+  private final ObjectMapper objectMapper;
+  private final JavaType javaType;
+  private final DeserializationConfig deserConfig;
+  private final AnnotationIntrospector ai;
+
+  JacksonTypeHelper(AnnotatedField field, ObjectMapper objectMapper) {
+    this.field = field;
+    this.objectMapper = objectMapper;
+    this.javaType = field.getType();
+    this.deserConfig = objectMapper.getDeserializationConfig();
+    this.ai = deserConfig.getAnnotationIntrospector();
+  }
+
+  JavaType type() {
+    if (ai == null || javaType == null || javaType.hasRawClass(Object.class)) {
+      return javaType;
+    } else {
+      try {
+        return ai.refineDeserializationType(deserConfig, field, javaType);
+      } catch (JsonMappingException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
+  ObjectWriter objectWriter() {
+    if (ai == null || javaType == null || javaType.hasRawClass(Object.class)) {
+      return objectMapper.writerFor(javaType);
+    } else {
+      try {
+        JavaType serType = ai.refineSerializationType(objectMapper.getSerializationConfig(), field, javaType);
+        return objectMapper.writerFor(serType);
+      } catch (JsonMappingException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+}

ebean-core/src/main/java/io/ebeaninternal/server/type/RsetDataReader.java

@@ -43,12 +43,8 @@ public class RsetDataReader implements DataReader {
 
   @Override
   public boolean next() throws SQLException {
-    return rset.next();
-  }
-
-  @Override
-  public void resetColumnPosition() {
     pos = 0;
+    return rset.next();
   }
 
   @Override
@@ -75,7 +71,6 @@ public class RsetDataReader implements DataReader {
     return rset.getBigDecimal(pos());
   }
 
-
   @Override
   public InputStream getBinaryStream() throws SQLException {
     return rset.getBinaryStream(pos());
@@ -84,19 +79,13 @@ public class RsetDataReader implements DataReader {
   @Override
   public Boolean getBoolean() throws SQLException {
     boolean v = rset.getBoolean(pos());
-    if (rset.wasNull()) {
-      return null;
-    }
-    return v;
+    return rset.wasNull() ? null : v;
   }
 
   @Override
   public Byte getByte() throws SQLException {
     byte v = rset.getByte(pos());
-    if (rset.wasNull()) {
-      return null;
-    }
-    return v;
+    return rset.wasNull() ? null : v;
   }
 
   @Override
@@ -117,62 +106,42 @@ public class RsetDataReader implements DataReader {
   @Override
   public Double getDouble() throws SQLException {
     double v = rset.getDouble(pos());
-    if (rset.wasNull()) {
-      return null;
-    }
-    return v;
+    return rset.wasNull() ? null : v;
   }
 
   @Override
   public Float getFloat() throws SQLException {
     float v = rset.getFloat(pos());
-    if (rset.wasNull()) {
-      return null;
-    }
-    return v;
+    return rset.wasNull() ? null : v;
   }
 
   @Override
   public Integer getInt() throws SQLException {
     int v = rset.getInt(pos());
-    if (rset.wasNull()) {
-      return null;
-    }
-    return v;
+    return rset.wasNull() ? null : v;
   }
 
-
   @Override
   public Long getLong() throws SQLException {
     long v = rset.getLong(pos());
-    if (rset.wasNull()) {
-      return null;
-    }
-    return v;
+    return rset.wasNull() ? null : v;
   }
 
-
   public Ref getRef() throws SQLException {
     return rset.getRef(pos());
   }
 
-
   @Override
   public Short getShort() throws SQLException {
     short s = rset.getShort(pos());
-    if (rset.wasNull()) {
-      return null;
-    }
-    return s;
+    return rset.wasNull() ? null : s;
   }
 
-
   @Override
   public String getString() throws SQLException {
     return rset.getString(pos());
   }
 
-
   @Override
   public Time getTime() throws SQLException {
     Calendar cal = dataTimeZone.getTimeZone();
@@ -196,14 +165,10 @@ public class RsetDataReader implements DataReader {
   @Override
   public String getStringFromStream() throws SQLException {
     Reader reader = rset.getCharacterStream(pos());
-    if (reader == null) {
-      return null;
-    }
-    return readStringLob(reader);
+    return reader == null ? null : readStringLob(reader);
   }
 
   protected String readStringLob(Reader reader) throws SQLException {
-
     char[] buffer = new char[clobBufferSize];
     int readLength;
     StringBuilder out = new StringBuilder(stringInitialSize);
@@ -215,7 +180,6 @@ public class RsetDataReader implements DataReader {
     } catch (IOException e) {
       throw new SQLException("IOException reading Clob " + e.getMessage());
     }
-
     return out.toString();
   }
 
@@ -226,19 +190,16 @@ public class RsetDataReader implements DataReader {
   }
 
   protected byte[] getBinaryLob(InputStream in) throws SQLException {
-
     if (in == null) {
       return null;
     }
     try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
-
       byte[] buf = new byte[bufferSize];
       int len;
       while ((len = in.read(buf, 0, buf.length)) != -1) {
         out.write(buf, 0, len);
       }
       byte[] data = out.toByteArray();
-
       if (data.length == 0) {
         data = null;
       }

ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypeJsonCollection.java

@@ -61,7 +61,7 @@ abstract class ScalarTypeJsonCollection<T> extends ScalarTypeBase<T> implements
    */
   @Override
   public boolean isDirty(Object value) {
-    return CheckMarkedDirty.isDirty(value);
+    return TypeJsonManager.checkIsDirty(value);
   }
 
   @Override

ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypeJsonMap.java

@@ -123,7 +123,7 @@ public abstract class ScalarTypeJsonMap extends ScalarTypeBase<Map> {
    */
   @Override
   public boolean isDirty(Object value) {
-    return CheckMarkedDirty.isDirty(value);
+    return TypeJsonManager.checkIsDirty(value);
   }
 
   @Override

ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypeJsonObjectMapper.java

@@ -3,15 +3,10 @@ package io.ebeaninternal.server.type;
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.AnnotationIntrospector;
-import com.fasterxml.jackson.databind.DeserializationConfig;
 import com.fasterxml.jackson.databind.JavaType;
-import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.SerializationConfig;
 import com.fasterxml.jackson.databind.introspect.AnnotatedField;
-import io.ebean.config.dbplatform.DbPlatformType;
 import io.ebean.core.type.DataBinder;
 import io.ebean.core.type.DataReader;
 import io.ebean.core.type.DocPropertyType;
@@ -34,38 +29,23 @@ import java.util.Set;
 /**
  * Supports @DbJson properties using Jackson ObjectMapper.
  */
-public class ScalarTypeJsonObjectMapper {
+class ScalarTypeJsonObjectMapper {
 
   /**
    * Create and return the appropriate ScalarType.
    */
-  public static ScalarType<?> createTypeFor(boolean postgres, AnnotatedField field, ObjectMapper objectMapper,
-                                            int dbType, DocPropertyType docType) {
-
+  static ScalarType<?> createTypeFor(TypeJsonManager jsonManager, AnnotatedField field, int dbType, DocPropertyType docType) {
     Class<?> type = field.getRawType();
-    String pgType = getPostgresType(postgres, dbType);
     if (Set.class.equals(type)) {
-      return new OmSet(objectMapper, field, dbType, pgType, docType);
+      return new OmSet(jsonManager, field, dbType, docType);
     }
     if (List.class.equals(type)) {
-      return new OmList(objectMapper, field, dbType, pgType, docType);
+      return new OmList(jsonManager, field, dbType, docType);
     }
     if (Map.class.equals(type)) {
-      return new OmMap(objectMapper, field, dbType, pgType);
+      return new OmMap(jsonManager, field, dbType);
     }
-    return new GenericObject(objectMapper, field, dbType, pgType);
-  }
-
-  private static String getPostgresType(boolean postgres, int dbType) {
-    if (postgres) {
-      switch (dbType) {
-        case DbPlatformType.JSON:
-          return PostgresHelper.JSON_TYPE;
-        case DbPlatformType.JSONB:
-          return PostgresHelper.JSONB_TYPE;
-      }
-    }
-    return null;
+    return new GenericObject(jsonManager, field, dbType, type);
   }
 
   /**
@@ -73,8 +53,8 @@ public class ScalarTypeJsonObjectMapper {
    */
   private static class GenericObject extends Base<Object> {
 
-    public GenericObject(ObjectMapper objectMapper, AnnotatedField field, int dbType, String pgType) {
-      super(Object.class, objectMapper, field, dbType, pgType, DocPropertyType.OBJECT);
+    GenericObject(TypeJsonManager jsonManager, AnnotatedField field, int dbType, Class<?> rawType) {
+      super(Object.class, jsonManager, field, dbType, DocPropertyType.OBJECT, rawType);
     }
   }
 
@@ -84,8 +64,8 @@ public class ScalarTypeJsonObjectMapper {
   @SuppressWarnings("rawtypes")
   private static class OmSet extends Base<Set> {
 
-    public OmSet(ObjectMapper objectMapper, AnnotatedField field, int dbType, String pgType, DocPropertyType docType) {
-      super(Set.class, objectMapper, field, dbType, pgType, docType);
+    OmSet(TypeJsonManager jsonManager, AnnotatedField field, int dbType, DocPropertyType docType) {
+      super(Set.class, jsonManager, field, dbType, docType);
     }
 
     @Override
@@ -102,8 +82,8 @@ public class ScalarTypeJsonObjectMapper {
   @SuppressWarnings("rawtypes")
   private static class OmList extends Base<List> {
 
-    public OmList(ObjectMapper objectMapper, AnnotatedField field, int dbType, String pgType, DocPropertyType docType) {
-      super(List.class, objectMapper, field, dbType, pgType, docType);
+    OmList(TypeJsonManager jsonManager, AnnotatedField field, int dbType, DocPropertyType docType) {
+      super(List.class, jsonManager, field, dbType, docType);
     }
 
     @Override
@@ -120,8 +100,8 @@ public class ScalarTypeJsonObjectMapper {
   @SuppressWarnings("rawtypes")
   private static class OmMap extends Base<Map> {
 
-    public OmMap(ObjectMapper objectMapper, AnnotatedField field, int dbType, String pgType) {
-      super(Map.class, objectMapper, field, dbType, pgType, DocPropertyType.OBJECT);
+    OmMap(TypeJsonManager jsonManager, AnnotatedField field, int dbType) {
+      super(Map.class, jsonManager, field, dbType, DocPropertyType.OBJECT);
     }
 
     @Override
@@ -139,51 +119,25 @@ public class ScalarTypeJsonObjectMapper {
   private static abstract class Base<T> extends ScalarTypeBase<T> {
 
     private final ObjectWriter objectWriter;
-
     private final ObjectMapper objectReader;
-
-    private JavaType deserType;
-
+    private final JavaType deserType;
     private final String pgType;
-
     private final DocPropertyType docType;
+    private final TypeJsonManager.DirtyHandler dirtyHandler;
 
-    /**
-     * Construct given the object mapper, property type and DB type for storage.
-     */
-    public Base(Class<T> cls, ObjectMapper objectMapper, AnnotatedField field, int dbType, String pgType, DocPropertyType docType) {
+    Base(Class<T> cls, TypeJsonManager jsonManager, AnnotatedField field, int dbType, DocPropertyType docType) {
+      this(cls, jsonManager, field, dbType, docType, cls);
+    }
+
+    Base(Class<T> cls, TypeJsonManager jsonManager, AnnotatedField field, int dbType, DocPropertyType docType, Class<?> rawType) {
       super(cls, false, dbType);
-      this.pgType = pgType;
+      this.objectReader = jsonManager.objectMapper();
+      this.pgType = jsonManager.postgresType(dbType);
       this.docType = docType;
-      this.objectReader = objectMapper;
-
-      JavaType javaType = field.getType();
-      DeserializationConfig deserConfig = objectMapper.getDeserializationConfig();
-      AnnotationIntrospector ai = deserConfig.getAnnotationIntrospector();
-
-      if (ai != null && javaType != null && !javaType.hasRawClass(Object.class)) {
-        try {
-          this.deserType = ai.refineDeserializationType(deserConfig, field, javaType);
-        } catch (JsonMappingException e) {
-          throw new RuntimeException(e);
-        }
-      } else {
-        this.deserType = javaType;
-      }
-
-      SerializationConfig serConfig = objectMapper.getSerializationConfig();
-       ai = deserConfig.getAnnotationIntrospector();
-
-       if (ai != null && javaType != null && !javaType.hasRawClass(Object.class)) {
-         try {
-           JavaType serType = ai.refineSerializationType(serConfig, field, javaType);
-           this.objectWriter = objectMapper.writerFor(serType);
-         } catch (JsonMappingException e) {
-           throw new RuntimeException(e);
-         }
-       } else {
-         this.objectWriter = objectMapper.writerFor(javaType);
-       }
+      this.dirtyHandler = jsonManager.dirtyHandler(cls, rawType);
+      final JacksonTypeHelper helper = new JacksonTypeHelper(field, objectReader);
+      this.deserType = helper.type();
+      this.objectWriter = helper.objectWriter();
     }
 
     /**
@@ -199,7 +153,7 @@ public class ScalarTypeJsonObjectMapper {
      */
     @Override
     public boolean isDirty(Object value) {
-      return CheckMarkedDirty.isDirty(value);
+      return dirtyHandler.isDirty(value);
     }
 
     @Override
@@ -224,12 +178,7 @@ public class ScalarTypeJsonObjectMapper {
         if (value == null) {
           binder.setNull(Types.VARCHAR); // use varchar, otherwise SqlServer/db2 will fail with 'Invalid JDBC data type 5.001.'
         } else {
-          try {
-            String json = objectWriter.writeValueAsString(value);
-            binder.setString(json);
-          } catch (JsonProcessingException e) {
-            throw new SQLException("Unable to create JSON", e);
-          }
+          binder.setString(formatValue(value));
         }
       }
     }

ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypeLocalDateTime.java

@@ -1,7 +1,10 @@
 package io.ebeaninternal.server.type;
 
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
 import io.ebean.config.JsonConfig;
 
+import java.io.IOException;
 import java.sql.Timestamp;
 import java.sql.Types;
 import java.time.Instant;
@@ -29,12 +32,37 @@ public class ScalarTypeLocalDateTime extends ScalarTypeBaseDateTime<LocalDateTim
 
   @Override
   protected String toJsonNanos(LocalDateTime value) {
-    return String.valueOf(convertToMillis(value));
+    return value.toString();
   }
 
   @Override
-  protected String toJsonISO8601(LocalDateTime dateTime) {
-    return dateTime.atZone(ZoneId.systemDefault()).toInstant().toString();
+  protected String toJsonISO8601(LocalDateTime value) {
+    return value.toString();
+  }
+
+  @Override
+  protected LocalDateTime fromJsonISO8601(String value) {
+    return LocalDateTime.parse(value);
+  }
+
+  @Override
+  public LocalDateTime jsonRead(JsonParser parser) throws IOException {
+    return LocalDateTime.parse(parser.getText());
+  }
+
+  @Override
+  public void jsonWrite(JsonGenerator writer, LocalDateTime value) throws IOException {
+    writer.writeString(value.toString());
+  }
+
+  @Override
+  public String formatValue(LocalDateTime value) {
+    return value.toString();
+  }
+
+  @Override
+  public LocalDateTime parse(String value) {
+    return LocalDateTime.parse(value);
   }
 
   @Override

ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypeOffsetDateTime.java

@@ -15,8 +15,11 @@ import static io.ebeaninternal.server.type.IsoJsonDateTimeParser.formatIso;
  */
 public class ScalarTypeOffsetDateTime extends ScalarTypeBaseDateTime<OffsetDateTime> {
 
-  public ScalarTypeOffsetDateTime(JsonConfig.DateTime mode) {
+  private final ZoneId zoneId;
+
+  public ScalarTypeOffsetDateTime(JsonConfig.DateTime mode, ZoneId zoneId) {
     super(mode, OffsetDateTime.class, false, Types.TIMESTAMP);
+    this.zoneId = zoneId;
   }
 
   @Override
@@ -46,7 +49,7 @@ public class ScalarTypeOffsetDateTime extends ScalarTypeBaseDateTime<OffsetDateT
 
   @Override
   public OffsetDateTime convertFromInstant(Instant ts) {
-    return OffsetDateTime.ofInstant(ts, ZoneId.systemDefault());
+    return OffsetDateTime.ofInstant(ts, zoneId);
   }
 
   @Override

ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypePostgresHstore.java

@@ -33,7 +33,7 @@ public class ScalarTypePostgresHstore extends ScalarTypeBase<Map> {
 
   @Override
   public boolean isDirty(Object value) {
-    return CheckMarkedDirty.isDirty(value);
+    return TypeJsonManager.checkIsDirty(value);
   }
 
   @SuppressWarnings("unchecked")

ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypeZonedDateTime.java

@@ -13,8 +13,11 @@ import java.time.ZonedDateTime;
  */
 public class ScalarTypeZonedDateTime extends ScalarTypeBaseDateTime<ZonedDateTime> {
 
-  public ScalarTypeZonedDateTime(JsonConfig.DateTime mode) {
+  private final ZoneId zoneId;
+
+  public ScalarTypeZonedDateTime(JsonConfig.DateTime mode, ZoneId zoneId) {
     super(mode, ZonedDateTime.class, false, Types.TIMESTAMP);
+    this.zoneId = zoneId;
   }
 
   @Override
@@ -44,7 +47,7 @@ public class ScalarTypeZonedDateTime extends ScalarTypeBaseDateTime<ZonedDateTim
 
   @Override
   public ZonedDateTime convertFromInstant(Instant ts) {
-    return ZonedDateTime.ofInstant(ts, ZoneId.systemDefault());
+    return ZonedDateTime.ofInstant(ts, zoneId);
   }
 
   @Override

ebean-core/src/main/java/io/ebeaninternal/server/type/TypeJsonManager.java

@@ -0,0 +1,98 @@
+package io.ebeaninternal.server.type;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.ebean.ModifyAwareType;
+import io.ebean.config.DatabaseConfig;
+import io.ebean.config.dbplatform.DbPlatformType;
+
+class TypeJsonManager {
+
+  interface DirtyHandler {
+    boolean isDirty(Object value);
+  }
+
+  private final boolean postgres;
+  private final ObjectMapper objectMapper;
+  private final DirtyHandler defaultHandler;
+  private final DirtyHandler modifyAwareHandler;
+
+  TypeJsonManager(boolean postgres, Object objectMapper, boolean defaultDirty) {
+    this.postgres = postgres;
+    this.objectMapper = (ObjectMapper) objectMapper;
+    this.defaultHandler = new DefaultHandler(defaultDirty);
+    this.modifyAwareHandler = new ModifyAwareHandler();
+  }
+
+  ObjectMapper objectMapper() {
+    return objectMapper;
+  }
+
+  String postgresType(int dbType) {
+    if (postgres) {
+      switch (dbType) {
+        case DbPlatformType.JSON:
+          return PostgresHelper.JSON_TYPE;
+        case DbPlatformType.JSONB:
+          return PostgresHelper.JSONB_TYPE;
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Return the DirtyHandler to use.
+   */
+  DirtyHandler dirtyHandler(Class<?> cls, Class<?> rawType) {
+    if (!Object.class.equals(cls) || ModifyAwareType.class.isAssignableFrom(rawType)) {
+      // Set, List and Map are modify aware
+      return modifyAwareHandler;
+    }
+    return defaultHandler;
+  }
+
+  /**
+   * Return true if the value should be considered dirty (and included in an update).
+   */
+  static boolean checkIsDirty(Object value) {
+    if (value instanceof ModifyAwareType) {
+      return checkModifyAware(value);
+    }
+    return true;
+  }
+
+  private static boolean checkModifyAware(Object value) {
+    ModifyAwareType modifyAware = (ModifyAwareType) value;
+    if (modifyAware.isMarkedDirty()) {
+      // reset the dirty state (consider not dirty after update)
+      modifyAware.setMarkedDirty(false);
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  static final class ModifyAwareHandler implements DirtyHandler {
+    @Override
+    public boolean isDirty(Object value) {
+      return checkModifyAware(value);
+    }
+  }
+
+  /**
+   * Effectively constant based on {@link DatabaseConfig#isJsonDirtyByDefault()}
+   */
+  static final class DefaultHandler implements DirtyHandler {
+
+    private final boolean dirty;
+
+    DefaultHandler(boolean dirty) {
+      this.dirty = dirty;
+    }
+
+    @Override
+    public boolean isDirty(Object value) {
+      return dirty;
+    }
+  }
+
+}

ebean-core/src/main/java/io/ebeaninternal/server/util/Str.java

@@ -1,4 +1,4 @@
-package io.ebeaninternal.server.lib;
+package io.ebeaninternal.server.util;
 
 /**
  * String utility for adding strings together.
@@ -11,7 +11,6 @@ public class Str {
    * Append strings together.
    */
   public static String add(String s0, String s1, String... args) {
-
     // determine a decent buffer size
     int len = 16 + s0.length();
     if (s1 != null) {

ebean-core/src/main/resources/META-INF/ebean-version.mf

@@ -0,0 +1 @@
+ebean-version: 129

ebean-core/src/test/java/io/ebean/DtoQuery2Test.java

@@ -6,10 +6,13 @@ import org.ebeantest.LoggedSqlCollector;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.tests.model.basic.Customer;
 import org.tests.model.basic.ResetBasicData;
 
 import java.util.List;
 import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -30,6 +33,73 @@ public class DtoQuery2Test extends BaseTestCase {
     assertThat(list).isNotEmpty();
   }
 
+  @Test
+  public void dto_findIterator_closeWithResources() {
+    ResetBasicData.reset();
+
+    int counter = 0;
+    try (QueryIterator<DCust> iterator = server()
+      .findDto(DCust.class, "select id, name from o_customer where id > ?")
+      .setParameter(0)
+      .findIterate()) {
+
+      if (iterator.hasNext()) {
+        counter++;
+      }
+    }
+
+    assertThat(counter).isEqualTo(1);
+  }
+
+  @Test
+  public void dto_findIterator() {
+    ResetBasicData.reset();
+    final int expectedCount = server().find(Customer.class).findCount();
+
+    LoggedSqlCollector.start();
+    int counter = 0;
+    try (final QueryIterator<DCust> iterator = server().findDto(DCust.class, "select id, name from o_customer where id > :id")
+      .setParameter("id", 0)
+      .findIterate()) {
+
+      while (iterator.hasNext()) {
+        final DCust cust = iterator.next();
+        counter++;
+        assertThat(cust).isNotNull();
+        assertThat(cust.getName()).isNotNull();
+      }
+    }
+
+    assertThat(counter).isEqualTo(expectedCount);
+
+    List<String> sql = LoggedSqlCollector.stop();
+    assertSql(sql.get(0)).contains("select id, name from o_customer where id > ?");
+  }
+
+  @Test
+  public void dto_findStream() {
+    ResetBasicData.reset();
+    final int expectedCount = server().find(Customer.class).findCount();
+
+    LoggedSqlCollector.start();
+
+    try (final Stream<DCust> stream =
+           server()
+             .findDto(DCust.class, "select id, name from o_customer where id > ?")
+             .setParameter(0)
+             .findStream()) {
+
+      final List<String> names = stream
+        .map(DCust::getName)
+        .collect(Collectors.toList());
+
+      assertThat(names.size()).isEqualTo(expectedCount);
+    }
+
+    List<String> sql = LoggedSqlCollector.stop();
+    assertSql(sql.get(0)).contains("select id, name from o_customer where id > ?");
+  }
+
   @Test
   public void dto_findEach_constructorMatch() {
 

ebean-core/src/test/java/io/ebean/common/BeanMapTest.java

@@ -2,18 +2,21 @@ package io.ebean.common;
 
 import io.ebean.bean.BeanCollection;
 import org.junit.Test;
+import org.tests.model.basic.EBasic;
 
-import java.util.LinkedHashMap;
-import java.util.Map;
+import java.util.*;
 
+import static java.util.Arrays.asList;
 import static org.assertj.core.api.Assertions.assertThat;
 
 
 public class BeanMapTest {
 
-  Object object1 = new Object();
-  Object object2 = new Object();
-  Object object3 = new Object();
+  private final EBasic object1 = new EBasic("o1");
+  private final EBasic object2 = new EBasic("o2");
+  private final EBasic object3 = new EBasic("o3");
+  private final EBasic object4 = new EBasic("o4");
+  private final EBasic object5 = new EBasic("o5");
 
   private Map<String, Object> all() {
     Map<String, Object> all = new LinkedHashMap<>();
@@ -174,9 +177,7 @@ public class BeanMapTest {
   @Test
   public void testClear_given_someBeansInAdditions() throws Exception {
 
-    BeanMap<String, Object> map = new BeanMap<>();
-    map.put("1", object1);
-    map.setModifyListening(BeanCollection.ModifyListenMode.ALL);
+    BeanMap<String, EBasic> map = newModifyListeningMap();
     map.put("2", object2);
     map.put("3", object3);
 
@@ -188,4 +189,252 @@ public class BeanMapTest {
     assertThat(map.getModifyAdditions()).isEmpty();
   }
 
+  @Test(expected = UnsupportedOperationException.class)
+  public void keySet_add_whenModifyListening() {
+    BeanMap<String, EBasic> map = newModifyListeningMap();
+    map.keySet().add("3");
+  }
+
+  @Test(expected = UnsupportedOperationException.class)
+  public void keySet_add() {
+    BeanMap<String, Object> map = new BeanMap<>();
+    map.keySet().add("3");
+  }
+
+  @Test(expected = UnsupportedOperationException.class)
+  public void keySet_addAll_whenModifyListening() {
+    BeanMap<String, EBasic> map = newModifyListeningMap();
+    map.keySet().addAll(asList("3", "4"));
+  }
+
+  @Test(expected = UnsupportedOperationException.class)
+  public void keySet_addAll() {
+    BeanMap<String, Object> map = new BeanMap<>();
+    map.keySet().addAll(asList("3", "4"));
+  }
+
+  @Test
+  public void keySet_remove() {
+
+    BeanMap<String, Object> map = new BeanMap<>();
+    map.put("1", object1);
+    map.put("2", object2);
+    map.setModifyListening(BeanCollection.ModifyListenMode.ALL);
+
+    final Set<String> keySet = map.keySet();
+    keySet.remove("1");
+
+    assertThat(keySet.contains("1")).isFalse();
+    assertThat(map).doesNotContainKeys("1");
+    assertThat(map.get("1")).isNull();
+
+    assertThat(map.getModifyRemovals()).containsExactly(object1);
+  }
+
+  @Test
+  public void keySet_clear() {
+
+    BeanMap<String, Object> map = new BeanMap<>();
+    map.put("1", object1);
+    map.put("2", object2);
+    map.setModifyListening(BeanCollection.ModifyListenMode.ALL);
+
+    final Set<String> keySet = map.keySet();
+    keySet.clear();
+
+    assertThat(map).isEmpty();
+    assertThat(keySet).isEmpty();
+
+    assertThat(map.getModifyRemovals()).containsExactly(object1, object2);
+  }
+
+  @Test
+  public void keySet_iterator_remove() {
+
+    BeanMap<String, Object> map = new BeanMap<>();
+    map.put("1", object1);
+    map.put("2", object2);
+    map.put("3", object3);
+    map.setModifyListening(BeanCollection.ModifyListenMode.ALL);
+
+    final Set<String> keySet = map.keySet();
+    final Iterator<String> iterator = keySet.iterator();
+    while (iterator.hasNext()) {
+      final String key = iterator.next();
+      if (key.equals("2")) {
+        iterator.remove();
+      }
+    }
+
+    assertThat(map).hasSize(2);
+    assertThat(keySet).hasSize(2);
+    assertThat(keySet).containsExactly("1", "3");
+    assertThat(map).containsKeys("1", "3");
+
+    assertThat(map.getModifyRemovals()).containsExactly(object2);
+  }
+
+  @Test
+  public void keySet_removeAll() {
+
+    BeanMap<String, Object> map = new BeanMap<>();
+    map.put("1", object1);
+    map.put("2", object2);
+    map.put("3", object3);
+    map.put("4", object4);
+    map.put("5", object5);
+    map.setModifyListening(BeanCollection.ModifyListenMode.ALL);
+
+    final Set<String> keySet = map.keySet();
+    final boolean changed = keySet.removeAll(asList("2", "3", "5"));
+
+    assertThat(changed).isTrue();
+    assertThat(map).hasSize(2);
+    assertThat(keySet).hasSize(2);
+    assertThat(keySet).containsExactly("1", "4");
+    assertThat(map).containsKeys("1", "4");
+
+    assertThat(map.getModifyRemovals()).containsExactly(object2, object3, object5);
+  }
+
+
+  @Test
+  public void keySet_retainAll() {
+
+    BeanMap<String, Object> map = new BeanMap<>();
+    map.put("1", object1);
+    map.put("2", object2);
+    map.put("3", object3);
+    map.put("4", object4);
+    map.put("5", object5);
+    map.setModifyListening(BeanCollection.ModifyListenMode.ALL);
+
+    final Set<String> keySet = map.keySet();
+    final boolean changed = keySet.retainAll(asList("2", "3", "5"));
+
+    assertThat(changed).isTrue();
+    assertThat(map).hasSize(3);
+    assertThat(keySet).hasSize(3);
+    assertThat(keySet).containsExactly("2", "3", "5");
+    assertThat(map).containsKeys("2", "3", "5");
+
+    assertThat(map.getModifyRemovals()).containsExactly(object1, object4);
+  }
+
+  @Test(expected = UnsupportedOperationException.class)
+  public void values_add() {
+    BeanMap<String, EBasic> map = new BeanMap<>();
+    map.values().add(object3);
+  }
+
+  @Test(expected = UnsupportedOperationException.class)
+  public void values_addAll() {
+    BeanMap<String, EBasic> map = new BeanMap<>();
+    map.values().addAll(asList(object3, object5));
+  }
+
+  @Test(expected = UnsupportedOperationException.class)
+  public void entrySet_add() {
+    newModifyListeningMap()
+      .entrySet()
+      .add(new AbstractMap.SimpleEntry<>("3", object3));
+  }
+
+  @Test
+  public void entrySet_clear() {
+    final BeanMap<String, EBasic> map = newModifyListeningMap();
+    final Set<Map.Entry<String, EBasic>> entries = map.entrySet();
+    entries.clear();
+
+    assertThat(entries).isEmpty();
+    assertThat(map).isEmpty();
+    assertThat(map.getModifyRemovals()).containsExactly(object1);
+  }
+
+  @Test
+  public void entrySet_remove() {
+    final BeanMap<String, EBasic> map = newModifyListeningMap5();
+    final Set<Map.Entry<String, EBasic>> entries = map.entrySet();
+
+    assertThat(map).hasSize(5);
+
+    final boolean existed1 = entries.remove(new AbstractMap.SimpleEntry<>("1", object1));
+    assertThat(existed1).isTrue();
+
+    final boolean existed22 = entries.remove(new AbstractMap.SimpleEntry<>("22", object1));
+    assertThat(existed22).isFalse();
+
+    assertThat(map).hasSize(4);
+    assertThat(map.getModifyRemovals()).containsExactly(object1);
+  }
+
+  @Test
+  public void entrySet_remove_whenNotEqualValue() {
+    final BeanMap<String, EBasic> map = newModifyListeningMap5();
+    final Set<Map.Entry<String, EBasic>> entries = map.entrySet();
+
+    assertThat(map).hasSize(5);
+
+    final boolean modified = entries.remove(new AbstractMap.SimpleEntry<>("1", object2));
+    assertThat(modified).isFalse();
+
+    assertThat(map).hasSize(5);
+    assertThat(map.getModifyRemovals()).isNull();
+  }
+
+  @Test
+  public void entrySet_iterator_remove() {
+    final BeanMap<String, EBasic> map = newModifyListeningMap5();
+    final Set<Map.Entry<String, EBasic>> entries = map.entrySet();
+    final Iterator<Map.Entry<String, EBasic>> iterator = entries.iterator();
+    while (iterator.hasNext()) {
+      final Map.Entry<String, EBasic> entry = iterator.next();
+      if (entry.getKey().equals("2") || entry.getKey().equals("5")) {
+        iterator.remove();
+      }
+    }
+    assertThat(map).hasSize(3);
+    assertThat(entries).hasSize(3);
+    assertThat(map.getModifyRemovals()).containsExactly(object2, object5);
+  }
+
+  @Test
+  public void entrySet_removeAll() {
+    final BeanMap<String, EBasic> map = newModifyListeningMap5();
+    final Set<Map.Entry<String, EBasic>> entries = map.entrySet();
+
+    entries.removeAll(asList(new AbstractMap.SimpleEntry<>("1", object1), new AbstractMap.SimpleEntry<>("3", object4), new AbstractMap.SimpleEntry<>("4", object4)));
+    assertThat(map).hasSize(3);
+    assertThat(entries).hasSize(3);
+    assertThat(map.getModifyRemovals()).containsExactly(object1, object4);
+  }
+
+  @Test
+  public void entrySet_retainAll() {
+    final BeanMap<String, EBasic> map = newModifyListeningMap5();
+    final Set<Map.Entry<String, EBasic>> entries = map.entrySet();
+
+    entries.retainAll(asList(new AbstractMap.SimpleEntry<>("1", object1), new AbstractMap.SimpleEntry<>("3", object4), new AbstractMap.SimpleEntry<>("4", object4)));
+    assertThat(map).hasSize(2);
+    assertThat(entries).hasSize(2);
+    assertThat(map.getModifyRemovals()).containsExactly(object2, object3, object5);
+  }
+
+  private BeanMap<String, EBasic> newModifyListeningMap() {
+    BeanMap<String, EBasic> map = new BeanMap<>();
+    map.put("1", object1);
+    map.setModifyListening(BeanCollection.ModifyListenMode.ALL);
+    return map;
+  }
+
+  private BeanMap<String, EBasic> newModifyListeningMap5() {
+    BeanMap<String, EBasic> map = new BeanMap<>();
+    map.put("1", object1);
+    map.put("2", object2);
+    map.put("3", object3);
+    map.put("4", object4);
+    map.put("5", object5);
+    map.setModifyListening(BeanCollection.ModifyListenMode.ALL);
+    return map;
+  }
 }

ebean-core/src/test/java/io/ebean/config/ServerConfigTest.java

@@ -62,6 +62,7 @@ public class ServerConfigTest {
     props.setProperty("dbOffline", "true");
     props.setProperty("jsonDateTime", "MILLIS");
     props.setProperty("jsonDate", "MILLIS");
+    props.setProperty("jsonDirtyByDefault", "false");
     props.setProperty("autoReadOnlyDataSource", "true");
     props.setProperty("disableL2Cache", "true");
     props.setProperty("notifyL2CacheInForeground", "true");
@@ -103,6 +104,9 @@ public class ServerConfigTest {
     assertEquals(PlatformConfig.DbUuid.BINARY, serverConfig.getPlatformConfig().getDbUuid());
     assertEquals(JsonConfig.DateTime.MILLIS, serverConfig.getJsonDateTime());
     assertEquals(JsonConfig.Date.MILLIS, serverConfig.getJsonDate());
+    assertFalse(serverConfig.isJsonDirtyByDefault());
+    serverConfig.setJsonDirtyByDefault(true);
+    assertTrue(serverConfig.isJsonDirtyByDefault());
 
     assertEquals("r0,users,orgs", serverConfig.getEnabledL2Regions());
 
@@ -155,6 +159,7 @@ public class ServerConfigTest {
     assertFalse(serverConfig.isIdGeneratorAutomatic());
     assertEquals(JsonConfig.DateTime.ISO8601, serverConfig.getJsonDateTime());
     assertEquals(JsonConfig.Date.ISO8601, serverConfig.getJsonDate());
+    assertTrue(serverConfig.isJsonDirtyByDefault());
     assertTrue(serverConfig.getPlatformConfig().isCaseSensitiveCollation());
     assertTrue(serverConfig.isAutoLoadModuleInfo());
 

ebean-core/src/test/java/io/ebean/config/dbplatform/OraclePlatformTest.java

@@ -1,6 +1,7 @@
 package io.ebean.config.dbplatform;
 
 import io.ebean.config.PlatformConfig;
+import io.ebean.config.dbplatform.oracle.Oracle11Platform;
 import io.ebean.config.dbplatform.oracle.OraclePlatform;
 import org.junit.Test;
 
@@ -8,17 +9,27 @@ import static org.assertj.core.api.Assertions.assertThat;
 
 public class OraclePlatformTest {
 
+  @Test
+  public void columnAliasPrefix_Oracle11Platform() {
+    Oracle11Platform platform11 = new Oracle11Platform();
+    assertThat(platform11.columnAliasPrefix).isEqualTo("c");
+  }
+
+  @Test
+  public void columnAliasPrefix_OraclePlatform() {
+    OraclePlatform platform = new OraclePlatform();
+    assertThat(platform.columnAliasPrefix).isEqualTo("c");
+  }
+
   @Test
   public void uuid_default() {
 
     OraclePlatform platform = new OraclePlatform();
     platform.configure(new PlatformConfig(), false);
     DbPlatformType dbType = platform.getDbTypeMap().get(DbPlatformType.UUID);
-
     assertThat(dbType.renderType(0, 0)).isEqualTo("varchar2(40)");
   }
 
-
   @Test
   public void uuid_as_binary() {
 

ebean-core/src/test/java/io/ebeaninternal/api/TDSpiEbeanServer.java

@@ -453,6 +453,16 @@ public class TDSpiEbeanServer implements SpiEbeanServer {
   public <T> void findDtoEachWhile(SpiDtoQuery<T> query, Predicate<T> consumer) {
   }
 
+  @Override
+  public <T> QueryIterator<T> findDtoIterate(SpiDtoQuery<T> query) {
+    return null;
+  }
+
+  @Override
+  public <T> Stream<T> findDtoStream(SpiDtoQuery<T> query) {
+    return null;
+  }
+
   @Override
   public <T> List<T> findDtoList(SpiDtoQuery<T> query) {
     return null;
@@ -493,6 +503,10 @@ public class TDSpiEbeanServer implements SpiEbeanServer {
     return null;
   }
 
+  @Override
+  public <T> void findSingleAttributeEach(SpiSqlQuery query, Class<T> cls, Consumer<T> consumer) {
+  }
+
   @Override
   public <T> T findOneMapper(SpiSqlQuery query, RowMapper<T> mapper) {
     return null;

ebean-core/src/test/java/io/ebeaninternal/server/core/DefaultServer_createOrmQueryRequestTest.java

@@ -288,7 +288,7 @@ public class DefaultServer_createOrmQueryRequestTest extends BaseTestCase {
   }
 
   @Test
-  public void test_removeJoinToMany_when_filterMany() {
+  public void test_filterMany_included() {
 
     Query<Order> query = Ebean.find(Order.class)
       .fetch("details")
@@ -301,6 +301,40 @@ public class DefaultServer_createOrmQueryRequestTest extends BaseTestCase {
     OrmQueryRequest<Order> queryRequest = queryRequest(query);
     OrmQueryDetail detail = queryRequest.getQuery().getDetail();
 
+    assertThat(detail.getFetchPaths()).containsExactly("details", "details.product", "customer");
+  }
+
+  @Test
+  public void test_filterMany_excludedByOrdering() {
+
+    Query<Order> query = Ebean.find(Order.class)
+      .fetch("customer")
+      .fetch("customer.contacts")
+      .fetch("details")
+      .fetch("details.product")
+      .filterMany("details").eq("orderQuantity", 10)
+      .query();
+
+    OrmQueryRequest<Order> queryRequest = queryRequest(query);
+    OrmQueryDetail detail = queryRequest.getQuery().getDetail();
+
+    assertThat(detail.getFetchPaths()).containsExactly("customer", "customer.contacts");
+  }
+
+  @Test
+  public void test_filterMany_excludedExplicitly() {
+
+    Query<Order> query = Ebean.find(Order.class)
+      .fetchQuery("details")
+      .fetch("details.product")
+      .fetch("customer")
+      .fetch("customer.contacts")
+      .filterMany("details").eq("orderQuantity", 10)
+      .query();
+
+    OrmQueryRequest<Order> queryRequest = queryRequest(query);
+    OrmQueryDetail detail = queryRequest.getQuery().getDetail();
+
     assertThat(detail.getFetchPaths()).containsExactly("customer", "customer.contacts");
   }
 

ebean-core/src/test/java/io/ebeaninternal/server/type/ScalarTypeLocalDateTimeTest.java

@@ -1,25 +1,28 @@
 package io.ebeaninternal.server.type;
 
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
 import io.ebean.config.JsonConfig;
 import org.junit.Test;
 
+import java.io.StringWriter;
 import java.sql.Timestamp;
 import java.time.LocalDateTime;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
 
 public class ScalarTypeLocalDateTimeTest {
 
-  ScalarTypeLocalDateTime type = new ScalarTypeLocalDateTime(JsonConfig.DateTime.MILLIS);
+  private final ScalarTypeLocalDateTime type = new ScalarTypeLocalDateTime(JsonConfig.DateTime.MILLIS);
+
+  private final JsonFactory factory = new JsonFactory();
 
   // warm up
-  LocalDateTime warmUp = LocalDateTime.now();
+  private final LocalDateTime warmUp = LocalDateTime.now();
 
   @Test
-  public void testNowToMillis() throws Exception {
+  public void testNowToMillis() {
 
     warmUp.hashCode();
 
@@ -29,7 +32,7 @@ public class ScalarTypeLocalDateTimeTest {
   }
 
   @Test
-  public void testConvertToMillis() throws Exception {
+  public void testConvertToMillis() {
 
     LocalDateTime now = LocalDateTime.now().withNano(123_000_000); // jdk11 workaround
     long asMillis = type.convertToMillis(now);
@@ -39,7 +42,7 @@ public class ScalarTypeLocalDateTimeTest {
   }
 
   @Test
-  public void testConvertFromTimestamp() throws Exception {
+  public void testConvertFromTimestamp() {
 
     Timestamp now = new Timestamp(System.currentTimeMillis());
 
@@ -74,6 +77,24 @@ public class ScalarTypeLocalDateTimeTest {
     assertEquals(timestamp, timestamp1);
   }
 
+
+  @Test
+  public void testJsonRaw() throws Exception {
+
+    final LocalDateTime of = LocalDateTime.of(2020, 5, 4, 13, 20, 40);
+
+    ScalarTypeLocalDateTime typeIso = new ScalarTypeLocalDateTime(JsonConfig.DateTime.ISO8601);
+
+    StringWriter writer = new StringWriter();
+    JsonGenerator generator = factory.createGenerator(writer);
+
+    typeIso.jsonWrite(generator, of);
+    generator.flush();
+
+    assertThat(of.toString()).isEqualTo("2020-05-04T13:20:40");
+    assertThat(writer.toString()).isEqualTo("\"2020-05-04T13:20:40\"");
+  }
+
   @Test
   public void testJson() throws Exception {
 

ebean-core/src/test/java/io/ebeaninternal/server/type/ScalarTypeOffsetDateTimeTest.java

@@ -5,6 +5,8 @@ import org.junit.Test;
 
 import java.sql.Timestamp;
 import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.util.TimeZone;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
@@ -14,12 +16,12 @@ import static org.junit.Assert.assertTrue;
 public class ScalarTypeOffsetDateTimeTest {
 
 
-  ScalarTypeOffsetDateTime type = new ScalarTypeOffsetDateTime(JsonConfig.DateTime.MILLIS);
+  ScalarTypeOffsetDateTime type = new ScalarTypeOffsetDateTime(JsonConfig.DateTime.MILLIS, ZoneOffset.systemDefault());
 
   OffsetDateTime warmUp = OffsetDateTime.now();
 
   @Test
-  public void testConvertToMillis() throws Exception {
+  public void testConvertToMillis() {
 
     warmUp.hashCode();
 
@@ -30,7 +32,43 @@ public class ScalarTypeOffsetDateTimeTest {
   }
 
   @Test
-  public void testConvertFromTimestamp() throws Exception {
+  public void convertFromInstant_with_UTC_expect_matchingZoneOffset() {
+    final TimeZone timeZoneToUse = TimeZone.getTimeZone("UTC");
+    final ZoneOffset expectedZoneOffset = ZoneOffset.UTC;
+
+    convertFromInstantWithConfiguredTimeZone(timeZoneToUse, expectedZoneOffset);
+  }
+
+  @Test
+  public void convertFromInstant_with_EST_expect_matchingZoneOffset() {
+    final TimeZone timeZoneToUse = TimeZone.getTimeZone("EST");
+    final ZoneOffset expectedOffset = OffsetDateTime.now(timeZoneToUse.toZoneId()).getOffset();
+
+    convertFromInstantWithConfiguredTimeZone(timeZoneToUse, expectedOffset);
+  }
+
+  private void convertFromInstantWithConfiguredTimeZone(TimeZone timeZoneToUse, ZoneOffset expectedZoneOffset) {
+    TimeZone previous = TimeZone.getDefault();
+    try {
+      OffsetDateTime dateTime = OffsetDateTime.parse("2021-01-01T00:00:00+11:00");
+
+      // test ScalarTypeOffsetDateTime with the configured timeZone to use
+      ScalarTypeOffsetDateTime type = new ScalarTypeOffsetDateTime(JsonConfig.DateTime.MILLIS, timeZoneToUse.toZoneId());
+
+      // effectively we desire to ignore the system timezone and use the configured one
+      TimeZone.setDefault(timeZoneToUse);
+
+      final OffsetDateTime offsetDateTime = type.convertFromInstant(dateTime.toInstant());
+
+      assertEquals(expectedZoneOffset, offsetDateTime.getOffset());
+
+    } finally {
+      TimeZone.setDefault(previous);
+    }
+  }
+
+  @Test
+  public void testConvertFromTimestamp() {
 
     Timestamp now = new Timestamp(System.currentTimeMillis());
 
@@ -69,11 +107,11 @@ public class ScalarTypeOffsetDateTimeTest {
     JsonTester<OffsetDateTime> jsonTester = new JsonTester<>(type);
     jsonTester.test(now);
 
-    ScalarTypeOffsetDateTime typeNanos = new ScalarTypeOffsetDateTime(JsonConfig.DateTime.NANOS);
+    ScalarTypeOffsetDateTime typeNanos = new ScalarTypeOffsetDateTime(JsonConfig.DateTime.NANOS, ZoneOffset.systemDefault());
     jsonTester = new JsonTester<>(typeNanos);
     jsonTester.test(now);
 
-    ScalarTypeOffsetDateTime typeIso = new ScalarTypeOffsetDateTime(JsonConfig.DateTime.ISO8601);
+    ScalarTypeOffsetDateTime typeIso = new ScalarTypeOffsetDateTime(JsonConfig.DateTime.ISO8601, ZoneOffset.systemDefault());
     jsonTester = new JsonTester<>(typeIso);
     jsonTester.test(now);
   }
@@ -81,7 +119,7 @@ public class ScalarTypeOffsetDateTimeTest {
   @Test
   public void isoJsonFormatParse() {
 
-    ScalarTypeOffsetDateTime typeIso = new ScalarTypeOffsetDateTime(JsonConfig.DateTime.ISO8601);
+    ScalarTypeOffsetDateTime typeIso = new ScalarTypeOffsetDateTime(JsonConfig.DateTime.ISO8601, ZoneOffset.systemDefault());
 
     OffsetDateTime now = OffsetDateTime.now();
     String asJson = typeIso.toJsonISO8601(now);

ebean-core/src/test/java/io/ebeaninternal/server/type/ScalarTypeZonedDateTimeTest.java

@@ -4,7 +4,11 @@ import io.ebean.config.JsonConfig;
 import org.junit.Test;
 
 import java.sql.Timestamp;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
+import java.util.TimeZone;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.*;
@@ -12,12 +16,12 @@ import static org.junit.Assert.*;
 public class ScalarTypeZonedDateTimeTest {
 
 
-  ScalarTypeZonedDateTime type = new ScalarTypeZonedDateTime(JsonConfig.DateTime.MILLIS);
+  ScalarTypeZonedDateTime type = new ScalarTypeZonedDateTime(JsonConfig.DateTime.MILLIS, ZoneId.systemDefault());
 
   ZonedDateTime warmUp = ZonedDateTime.now();
 
   @Test
-  public void testConvertToMillis() throws Exception {
+  public void testConvertToMillis() {
 
     warmUp.hashCode();
 
@@ -29,7 +33,7 @@ public class ScalarTypeZonedDateTimeTest {
   }
 
   @Test
-  public void testConvertFromTimestamp() throws Exception {
+  public void testConvertFromTimestamp() {
 
     Timestamp now = new Timestamp(System.currentTimeMillis());
 
@@ -39,6 +43,41 @@ public class ScalarTypeZonedDateTimeTest {
     assertEquals(now, timestamp);
   }
 
+  @Test
+  public void convertFromInstant_with_UTC_expect_matchingZoneOffset() {
+    final TimeZone timeZoneToUse = TimeZone.getTimeZone("UTC");
+    final ZoneOffset expectedZoneOffset = ZoneOffset.UTC;
+
+    convertFromInstantWithConfiguredTimeZone(timeZoneToUse, expectedZoneOffset);
+  }
+
+  @Test
+  public void convertFromInstant_with_EST_expect_matchingZoneOffset() {
+    final TimeZone timeZoneToUse = TimeZone.getTimeZone("EST");
+    final ZoneOffset expectedOffset = OffsetDateTime.now(timeZoneToUse.toZoneId()).getOffset();
+
+    convertFromInstantWithConfiguredTimeZone(timeZoneToUse, expectedOffset);
+  }
+
+  private void convertFromInstantWithConfiguredTimeZone(TimeZone timeZoneToUse, ZoneOffset expectedZoneOffset) {
+    TimeZone previous = TimeZone.getDefault();
+    try {
+      OffsetDateTime dateTime = OffsetDateTime.parse("2021-01-01T00:00:00+11:00");
+
+      // test ScalarTypeOffsetDateTime with the configured timeZone to use
+      ScalarTypeZonedDateTime type = new ScalarTypeZonedDateTime(JsonConfig.DateTime.MILLIS, timeZoneToUse.toZoneId());
+
+      // effectively we desire to ignore the system timezone and use the configured one
+      TimeZone.setDefault(timeZoneToUse);
+
+      final ZonedDateTime zonedDateTime = type.convertFromInstant(dateTime.toInstant());
+
+      assertEquals(expectedZoneOffset, zonedDateTime.getOffset());
+
+    } finally {
+      TimeZone.setDefault(previous);
+    }
+  }
 
   @Test
   public void testToJdbcType() throws Exception {
@@ -68,11 +107,11 @@ public class ScalarTypeZonedDateTimeTest {
     JsonTester<ZonedDateTime> jsonTester = new JsonTester<>(type);
     jsonTester.test(now);
 
-    ScalarTypeZonedDateTime typeNanos = new ScalarTypeZonedDateTime(JsonConfig.DateTime.NANOS);
+    ScalarTypeZonedDateTime typeNanos = new ScalarTypeZonedDateTime(JsonConfig.DateTime.NANOS, ZoneId.systemDefault());
     jsonTester = new JsonTester<>(typeNanos);
     jsonTester.test(now);
 
-    ScalarTypeZonedDateTime typeIso = new ScalarTypeZonedDateTime(JsonConfig.DateTime.ISO8601);
+    ScalarTypeZonedDateTime typeIso = new ScalarTypeZonedDateTime(JsonConfig.DateTime.ISO8601, ZoneId.systemDefault());
     jsonTester = new JsonTester<>(typeIso);
     jsonTester.test(now);
   }
@@ -80,7 +119,7 @@ public class ScalarTypeZonedDateTimeTest {
   @Test
   public void toJsonISO8601() {
 
-    ScalarTypeZonedDateTime typeIso = new ScalarTypeZonedDateTime(JsonConfig.DateTime.ISO8601);
+    ScalarTypeZonedDateTime typeIso = new ScalarTypeZonedDateTime(JsonConfig.DateTime.ISO8601, ZoneId.systemDefault());
 
     ZonedDateTime now = ZonedDateTime.now();
     String asJson = typeIso.toJsonISO8601(now);

ebean-core/src/test/java/org/tests/basic/TestFetchId.java

@@ -1,10 +1,14 @@
 package org.tests.basic;
 
 import io.ebean.BaseTestCase;
+import io.ebean.DB;
 import io.ebean.Ebean;
 import io.ebean.FutureIds;
 import io.ebean.Query;
+import io.ebeantest.LoggedSql;
+
 import org.tests.model.basic.Order;
+import org.tests.model.basic.OrderDetail;
 import org.tests.model.basic.ResetBasicData;
 import org.junit.Test;
 
@@ -36,4 +40,36 @@ public class TestFetchId extends BaseTestCase {
     List<Object> idList = futureIds.get();
     assertThat(idList).isNotEmpty();
   }
+  
+  @Test
+  public void testFetchIdWithExists() throws InterruptedException, ExecutionException {
+
+    ResetBasicData.reset();
+
+    Query<OrderDetail> subQuery = Ebean.find(OrderDetail.class)
+        .alias("sq")
+        .where().raw("details.id = sq.id").query();
+    Query<Order> query = Ebean.find(Order.class)
+      .where().exists(subQuery)
+      .orderBy("orderDate").query();
+
+    List<Object> ids = query.findIds();
+    // TODO: assert(query.getGeneratedSql())
+    assertThat(ids).isNotEmpty();
+    FutureIds<Order> futureIds = query.findFutureIds();
+
+    // wait for all the id's to be fetched
+    List<Object> idList = futureIds.get();
+    assertThat(idList).isNotEmpty();
+  }
+  
+  @Test
+  public void testFetchIdWithOrderFormula() throws InterruptedException, ExecutionException {
+
+    ResetBasicData.reset();
+
+    Query<Order> query = DB.find(Order.class).orderBy("totalItems");
+    query.findIds();
+    // TODO: assert(query.getGeneratedSql())
+  }
 }