ADD: Formula properties can be used in orderBy and where clause

ebean-core/src/main/java/io/ebeaninternal/api/ManyWhereJoins.java

@@ -9,7 +9,9 @@ import io.ebeaninternal.server.query.SqlJoinType;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.TreeMap;
 import java.util.TreeSet;
 
@@ -22,7 +24,7 @@ public final class ManyWhereJoins implements Serializable {
   private static final long serialVersionUID = -6490181101871795417L;
 
   private final TreeMap<String, PropertyJoin> joins = new TreeMap<>();
-  private List<String> formulaJoinProperties;
+  private Map<String, List<String>> formulaJoinProperties;
   private boolean aggregation;
   /**
    * 'Mode' indicating that joins added while this is true are required to be outer joins.
@@ -121,25 +123,25 @@ public final class ManyWhereJoins implements Serializable {
    * In findCount query found a formula property with a join clause so building a select clause
    * specifically for the findCount query.
    */
-  public void addFormulaWithJoin(String propertyName) {
+  public void addFormulaWithJoin(String prefix, String name) {
     if (formulaJoinProperties == null) {
-      formulaJoinProperties = new ArrayList<>();
+      formulaJoinProperties = new HashMap<>();
     }
-    formulaJoinProperties.add(propertyName);
+    formulaJoinProperties.computeIfAbsent(prefix, k -> new ArrayList<>()).add(name);
   }
 
   /**
    * Return true if the query select includes a formula with join.
    */
-  public boolean isFormulaWithJoin() {
-    return formulaJoinProperties != null;
+  public boolean isFormulaWithJoin(String prefix) {
+    return formulaJoinProperties != null && formulaJoinProperties.containsKey(prefix);
   }
 
   /**
    * Return the formula properties to build the select clause for a findCount query.
    */
-  public List<String> getFormulaJoinProperties() {
-    return formulaJoinProperties;
+  public List<String> getFormulaJoinProperties(String prefix) {
+    return formulaJoinProperties.get(prefix);
   }
 
   /**

ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanProperty.java

@@ -457,9 +457,9 @@ public class BeanProperty implements ElPropertyValue, Property, STreeProperty {
    * operation except for a OneToOne exported.
    */
   @Override
-  public void appendFrom(DbSqlContext ctx, SqlJoinType joinType) {
+  public void appendFrom(DbSqlContext ctx, SqlJoinType joinType, String manyWhere) {
     if (formula && sqlFormulaJoin != null) {
-      ctx.appendFormulaJoin(sqlFormulaJoin, joinType);
+      ctx.appendFormulaJoin(sqlFormulaJoin, joinType, manyWhere);
     } else if (secondaryTableJoin != null) {
       String relativePrefix = ctx.getRelativePrefix(secondaryTableJoinPrefix);
       secondaryTableJoin.addJoin(joinType, relativePrefix, ctx);

ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanPropertyAssocOne.java

@@ -402,6 +402,7 @@ public class BeanPropertyAssocOne<T> extends BeanPropertyAssoc<T> implements STr
   /**
    * Return the bean cache value for this property using original values.
    */
+  @Override
   public Object getCacheDataValueOrig(EntityBeanIntercept ebi) {
     return cacheDataConvert(ebi.getOrigValue(propertyIndex));
   }
@@ -602,11 +603,11 @@ public class BeanPropertyAssocOne<T> extends BeanPropertyAssoc<T> implements STr
   }
 
   @Override
-  public void appendFrom(DbSqlContext ctx, SqlJoinType joinType) {
+  public void appendFrom(DbSqlContext ctx, SqlJoinType joinType, String manyWhere) {
     if (!isTransient && !primaryKeyExport) {
       localHelp.appendFrom(ctx, joinType);
       if (sqlFormulaJoin != null) {
-        ctx.appendFormulaJoin(sqlFormulaJoin, joinType);
+        ctx.appendFormulaJoin(sqlFormulaJoin, joinType, manyWhere);
       }
     }
   }

ebean-core/src/main/java/io/ebeaninternal/server/deploy/DbSqlContext.java

@@ -72,7 +72,7 @@ public interface DbSqlContext {
    * Append a Sql Formula join. This converts the "${ta}" keyword to the current
    * table alias.
    */
-  void appendFormulaJoin(String sqlFormulaJoin, SqlJoinType joinType);
+  void appendFormulaJoin(String sqlFormulaJoin, SqlJoinType joinType, String manyWhere);
 
   /**
    * Return the current content length.

ebean-core/src/main/java/io/ebeaninternal/server/deploy/DynamicPropertyBase.java

@@ -69,7 +69,7 @@ abstract class DynamicPropertyBase implements STreeProperty {
   }
 
   @Override
-  public void appendFrom(DbSqlContext ctx, SqlJoinType joinType) {
+  public void appendFrom(DbSqlContext ctx, SqlJoinType joinType, String manyWhere) {
     // do not add to from usually
   }
 

ebean-core/src/main/java/io/ebeaninternal/server/expression/AbstractExpression.java

@@ -78,7 +78,7 @@ abstract class AbstractExpression implements SpiExpression {
       if (elProp != null) {
         if (elProp.containsFormulaWithJoin()) {
           // for findCount query select clause
-          manyWhereJoin.addFormulaWithJoin(propertyName);
+          manyWhereJoin.addFormulaWithJoin(elProp.getElPrefix(), elProp.getName());
         }
         if (elProp.containsMany()) {
           // for findCount we join to a many property

ebean-core/src/main/java/io/ebeaninternal/server/query/DefaultDbSqlContext.java

@@ -214,9 +214,9 @@ final class DefaultDbSqlContext implements DbSqlContext {
   }
 
   @Override
-  public void appendFormulaJoin(String sqlFormulaJoin, SqlJoinType joinType) {
+  public void appendFormulaJoin(String sqlFormulaJoin, SqlJoinType joinType, String manyWhere) {
     // replace ${ta} place holder with the real table alias...
-    String tableAlias = tableAliasStack.peek();
+    String tableAlias = manyWhere == null ? tableAliasStack.peek() : getTableAliasManyWhere(manyWhere);
     String converted = sqlFormulaJoin.replace(tableAliasPlaceHolder, tableAlias);
     if (formulaJoins == null) {
       formulaJoins = new HashSet<>();

ebean-core/src/main/java/io/ebeaninternal/server/query/STreeProperty.java

@@ -89,6 +89,6 @@ public interface STreeProperty extends ScalarDataReader<Object> {
   /**
    * Append to the from clause.
    */
-  void appendFrom(DbSqlContext ctx, SqlJoinType joinType);
+  void appendFrom(DbSqlContext ctx, SqlJoinType joinType, String manyWhere);
 
 }

ebean-core/src/main/java/io/ebeaninternal/server/query/SqlTreeBuilder.java

@@ -10,6 +10,7 @@ import io.ebeaninternal.server.deploy.InheritInfo;
 import io.ebeaninternal.server.deploy.TableJoin;
 import io.ebeaninternal.server.querydefn.OrmQueryDetail;
 import io.ebeaninternal.server.querydefn.OrmQueryProperties;
+import io.ebeaninternal.server.rawsql.SpiRawSql;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -255,6 +256,13 @@ public final class SqlTreeBuilder {
     }
     extraProps.forEach(props::add);
 
+    if (!rawSql && manyWhereJoins.isFormulaWithJoin(prefix)) {
+      for (String property : manyWhereJoins.getFormulaJoinProperties(prefix)) {
+        final STreeProperty beanProperty = desc.findPropertyFromPath(property);
+        myJoinList.add(new SqlTreeNodeFormulaWhereJoin(beanProperty, SqlJoinType.OUTER, null));
+      }
+    }
+
     SqlTreeNode selectNode = buildNode(prefix, prop, desc, myJoinList, props);
     if (joinList != null) {
       joinList.add(selectNode);
@@ -275,11 +283,11 @@ public final class SqlTreeBuilder {
       STreePropertyAssoc beanProperty = (STreePropertyAssoc) desc.findPropertyFromPath(joinProp.getProperty());
       SqlTreeNodeManyWhereJoin nodeJoin = new SqlTreeNodeManyWhereJoin(joinProp.getProperty(), beanProperty, joinProp.getSqlJoinType(), temporalMode);
       myJoinList.add(nodeJoin);
-    }
-    if (manyWhereJoins.isFormulaWithJoin()) {
-      for (String property : manyWhereJoins.getFormulaJoinProperties()) {
-        STreeProperty beanProperty = desc.findPropertyFromPath(property);
-        myJoinList.add(new SqlTreeNodeFormulaWhereJoin(beanProperty, SqlJoinType.OUTER));
+      if (manyWhereJoins.isFormulaWithJoin(joinProp.getProperty())) {
+        for (String property : manyWhereJoins.getFormulaJoinProperties(joinProp.getProperty())) {
+          STreeProperty beanProperty2 = desc.findPropertyFromPath(SplitName.add(joinProp.getProperty(), property));
+          myJoinList.add(new SqlTreeNodeFormulaWhereJoin(beanProperty2, SqlJoinType.OUTER, joinProp.getProperty()));
+        }
       }
     }
   }
@@ -348,7 +356,7 @@ public final class SqlTreeBuilder {
 
     // look for predicateIncludes that are not in selectIncludes and add
     // them as extra joins to the query
-    IncludesDistiller extraJoinDistill = new IncludesDistiller(desc, selectIncludes, predicateIncludes, temporalMode);
+    IncludesDistiller extraJoinDistill = new IncludesDistiller(desc, selectIncludes, predicateIncludes, manyWhereJoins, temporalMode);
     Collection<SqlTreeNodeExtraJoin> extraJoins = extraJoinDistill.getExtraJoinRootNodes();
     if (!extraJoins.isEmpty()) {
       // add extra joins required to support predicates
@@ -570,15 +578,17 @@ public final class SqlTreeBuilder {
     private final Set<String> selectIncludes;
     private final Set<String> predicateIncludes;
     private final SpiQuery.TemporalMode temporalMode;
+    private final ManyWhereJoins manyWhereJoins;
 
     private final Map<String, SqlTreeNodeExtraJoin> joinRegister = new HashMap<>();
     private final Map<String, SqlTreeNodeExtraJoin> rootRegister = new HashMap<>();
 
     private IncludesDistiller(STreeType desc, Set<String> selectIncludes,
-                              Set<String> predicateIncludes, SpiQuery.TemporalMode temporalMode) {
+                              Set<String> predicateIncludes, ManyWhereJoins manyWhereJoins, SpiQuery.TemporalMode temporalMode) {
       this.desc = desc;
       this.selectIncludes = selectIncludes;
       this.predicateIncludes = predicateIncludes;
+      this.manyWhereJoins = manyWhereJoins;
       this.temporalMode = temporalMode;
     }
 
@@ -610,6 +620,14 @@ public final class SqlTreeBuilder {
       if (extraJoin != null) {
         // add the extra join...
 
+        // add many where joins
+        if (manyWhereJoins.isFormulaWithJoin(includeProp)) {
+          for (String property : manyWhereJoins.getFormulaJoinProperties(includeProp)) {
+            STreeProperty beanProperty = desc.findPropertyFromPath(SplitName.add(includeProp, property));
+            extraJoin.addChild(new SqlTreeNodeFormulaWhereJoin(beanProperty, SqlJoinType.OUTER, null));
+          }
+        }
+
         // find root of this extra join... linking back to the
         // parents (creating the tree) as it goes.
         SqlTreeNodeExtraJoin root = findExtraJoinRoot(includeProp, extraJoin);

ebean-core/src/main/java/io/ebeaninternal/server/query/SqlTreeNodeBean.java

@@ -577,7 +577,7 @@ class SqlTreeNodeBean implements SqlTreeNode {
   public void appendFrom(DbSqlContext ctx, SqlJoinType joinType) {
     if (nodeBeanProp != null && nodeBeanProp.isFormula()) {
       // add joins for formula beans
-      nodeBeanProp.appendFrom(ctx, joinType);
+      nodeBeanProp.appendFrom(ctx, joinType, null);
     }
     ctx.pushJoin(prefix);
     ctx.pushTableAlias(prefix);
@@ -589,7 +589,7 @@ class SqlTreeNodeBean implements SqlTreeNode {
 
     for (STreeProperty property : properties) {
       // usually nothing... except for 1-1 Exported
-      property.appendFrom(ctx, joinType);
+      property.appendFrom(ctx, joinType, null);
     }
 
     for (SqlTreeNode child : children) {

ebean-core/src/main/java/io/ebeaninternal/server/query/SqlTreeNodeExtraJoin.java

@@ -28,7 +28,7 @@ final class SqlTreeNodeExtraJoin implements SqlTreeNode {
   private final String prefix;
   private final boolean manyJoin;
   private final boolean pathContainsMany;
-  private List<SqlTreeNodeExtraJoin> children;
+  private List<SqlTreeNode> children;
 
   SqlTreeNodeExtraJoin(String prefix, STreePropertyAssoc assocBeanProperty, boolean pathContainsMany, SpiQuery.TemporalMode temporalMode) {
     this.prefix = prefix;
@@ -92,7 +92,7 @@ final class SqlTreeNodeExtraJoin implements SqlTreeNode {
     return prefix;
   }
 
-  public void addChild(SqlTreeNodeExtraJoin child) {
+  public void addChild(SqlTreeNode child) {
     if (children == null) {
       children = new ArrayList<>();
     }
@@ -145,7 +145,7 @@ final class SqlTreeNodeExtraJoin implements SqlTreeNode {
     if (!manyToMany) {
       if (assocBeanProperty.isFormula()) {
         // add joins for formula beans
-        assocBeanProperty.appendFrom(ctx, joinType);
+        assocBeanProperty.appendFrom(ctx, joinType, null);
       }
       joinType = assocBeanProperty.addJoin(joinType, prefix, ctx);
       if (!oneToOneExported && assocBeanProperty.isTargetSoftDelete() && temporalMode != SpiQuery.TemporalMode.SOFT_DELETED) {
@@ -154,13 +154,19 @@ final class SqlTreeNodeExtraJoin implements SqlTreeNode {
     }
 
     if (children != null) {
+      ctx.pushJoin(prefix);
+      ctx.pushTableAlias(prefix);
+
       if (manyJoin || pathContainsMany) {
         // if AUTO then make all descendants use OUTER JOIN
         joinType = joinType.autoToOuter();
       }
-      for (SqlTreeNodeExtraJoin child : children) {
+      for (SqlTreeNode child : children) {
         child.appendFrom(ctx, joinType);
       }
+
+      ctx.popTableAlias();
+      ctx.popJoin();
     }
   }
 

ebean-core/src/main/java/io/ebeaninternal/server/query/SqlTreeNodeFormulaWhereJoin.java

@@ -21,9 +21,15 @@ final class SqlTreeNodeFormulaWhereJoin implements SqlTreeNode {
    */
   private final SqlJoinType manyJoinType;
 
-  SqlTreeNodeFormulaWhereJoin(STreeProperty prop, SqlJoinType manyJoinType) {
+  /**
+   * This is a many where formula join
+   */
+  private final String manyWhere;
+
+  SqlTreeNodeFormulaWhereJoin(STreeProperty prop, SqlJoinType manyJoinType, String manyWhere) {
     this.nodeBeanProp = prop;
     this.manyJoinType = manyJoinType;
+    this.manyWhere = manyWhere;
   }
 
   @Override
@@ -68,7 +74,7 @@ final class SqlTreeNodeFormulaWhereJoin implements SqlTreeNode {
   public void appendFrom(DbSqlContext ctx, SqlJoinType currentJoinType) {
     // always use the join type as per this many where join
     // (OUTER for disjunction and otherwise INNER)
-    nodeBeanProp.appendFrom(ctx, manyJoinType);
+    nodeBeanProp.appendFrom(ctx, manyJoinType, manyWhere);
   }
 
   @Override

ebean-core/src/main/java/io/ebeaninternal/server/querydefn/DefaultOrmQuery.java

@@ -416,7 +416,7 @@ public final class DefaultOrmQuery<T> extends AbstractQuery implements SpiQuery<
       for (Property orderProperty : orderBy.getProperties()) {
         ElPropertyDeploy elProp = beanDescriptor.getElPropertyDeploy(orderProperty.getProperty());
         if (elProp != null && elProp.containsFormulaWithJoin()) {
-          manyWhereJoins.addFormulaWithJoin(orderProperty.getProperty());
+          manyWhereJoins.addFormulaWithJoin(elProp.getElPrefix(), elProp.getName());
         }
       }
     }

ebean-core/src/test/java/org/tests/query/joins/TestQueryJoinOnFormula.java

@@ -2,7 +2,9 @@ package org.tests.query.joins;
 
 import io.ebean.BaseTestCase;
 import io.ebean.Ebean;
+import io.ebean.Query;
 import org.tests.model.basic.Order;
+import org.tests.model.basic.OrderShipment;
 import org.tests.model.basic.ResetBasicData;
 import org.tests.model.family.ChildPerson;
 import org.tests.model.family.ParentPerson;
@@ -70,6 +72,92 @@ public class TestQueryJoinOnFormula extends BaseTestCase {
     assertSql(sql.get(0)).contains("select count(*) from ( select t0.id from o_order t0  left join (select order_id,");
   }
 
+  @Test
+  public void testOrderOnChainedFormulaProperty() {
+    // test that join to order.details is not included
+
+    // Tests if SqlTreeBuilder.IncludesDistiller.createExtraJoin appends formulaJoinProperties
+    Query<OrderShipment> shipQuery = Ebean.find(OrderShipment.class)
+      .select("id")
+      .order().asc("order.totalAmount");
+
+    shipQuery.findList();
+    assertThat(shipQuery.getGeneratedSql()).isEqualTo("select t0.id "
+      + "from or_order_ship t0 "
+      + "left join o_order t1 on t1.id = t0.order_id  "
+      + "left join (select order_id, count(*) as total_items, sum(order_qty*unit_price) as total_amount from o_order_detail group by order_id) z_bt1 on z_bt1.order_id = t1.id "
+      + "order by z_bt1.total_amount");
+  }
+
+  @Test
+  public void testWhereOnChainedFormulaProperty() {
+    // test that join to order.details is not included
+
+    // Tests if SqlTreeBuilder.IncludesDistiller.createExtraJoin appends formulaJoinProperties
+    Query<OrderShipment> shipQuery = Ebean.find(OrderShipment.class)
+      .select("id")
+      .where().isNotNull("order.totalAmount").query();
+
+    shipQuery.findList();
+    assertThat(shipQuery.getGeneratedSql()).isEqualTo("select t0.id "
+      + "from or_order_ship t0 "
+      + "left join o_order t1 on t1.id = t0.order_id  "
+      + "left join (select order_id, count(*) as total_items, sum(order_qty*unit_price) as total_amount from o_order_detail group by order_id) z_bt1 on z_bt1.order_id = t1.id "
+      + "where z_bt1.total_amount is not null");
+  }
+
+  @Test
+  public void testWhereOnChainedFormulaManyWhere() {
+    // test that join to order.details is not included
+
+    // Tests if SqlTreeBuilder.IncludesDistiller.createExtraJoin appends formulaJoinProperties
+    Query<OrderShipment> shipQuery = Ebean.find(OrderShipment.class)
+      .select("id")
+      .where().isNotNull("order.shipments.order.totalAmount").query();
+
+    shipQuery.findList();
+    assertThat(shipQuery.getGeneratedSql()).isEqualTo("select distinct t0.id "
+      + "from or_order_ship t0 "
+      + "join o_order u1 on u1.id = t0.order_id "
+      + "join or_order_ship u2 on u2.order_id = u1.id "
+      + "join o_order u3 on u3.id = u2.order_id  "
+      + "left join (select order_id, count(*) as total_items, sum(order_qty*unit_price) as total_amount from o_order_detail group by order_id) z_bu3 on z_bu3.order_id = u3.id "
+      + "where z_bu3.total_amount is not null");
+  }
+  @Test
+  public void testOrderOnChainedFormulaPropertyWithFetch() {
+
+    // Tests if SqlTreeBuilder.buildSelectChain appends formulaJoinProperties
+    Query<OrderShipment> shipQuery = Ebean.find(OrderShipment.class)
+      .select("id")
+      .fetch("order", "totalAmount")
+      .order().asc("order.totalAmount");
+
+    shipQuery.findList();
+    assertThat(shipQuery.getGeneratedSql()).isEqualTo("select t0.id, t1.id, z_bt1.total_amount "
+      + "from or_order_ship t0 "
+      + "left join o_order t1 on t1.id = t0.order_id  "
+      + "left join (select order_id, count(*) as total_items, sum(order_qty*unit_price) as total_amount from o_order_detail group by order_id) z_bt1 on z_bt1.order_id = t1.id "
+      + "order by z_bt1.total_amount");
+
+  }
+  @Test
+  public void testWhereOnChainedFormulaPropertyWithFetch() {
+    // Tests if SqlTreeBuilder.buildSelectChain appends formulaJoinProperties
+    Query<OrderShipment> shipQuery = Ebean.find(OrderShipment.class)
+      .select("id")
+      .fetch("order", "totalAmount")
+      .where().isNotNull("order.totalAmount").query();
+
+    shipQuery.findList();
+    assertThat(shipQuery.getGeneratedSql()).isEqualTo("select t0.id, t1.id, z_bt1.total_amount "
+      + "from or_order_ship t0 "
+      + "left join o_order t1 on t1.id = t0.order_id  "
+      + "left join (select order_id, count(*) as total_items, sum(order_qty*unit_price) as total_amount from o_order_detail group by order_id) z_bt1 on z_bt1.order_id = t1.id "
+      + "where z_bt1.total_amount is not null");
+
+  }
+
   @Test
   public void test_OrderFindCount_multiFormula() {
 
@@ -217,8 +305,10 @@ public class TestQueryJoinOnFormula extends BaseTestCase {
 
     List<String> loggedSql = LoggedSqlCollector.stop();
     assertEquals(1, loggedSql.size());
-    assertThat(loggedSql.get(0)).contains("select t0.identifier from child_person t0 left join (select i2.parent_identifier");
-    assertThat(loggedSql.get(0)).contains("where coalesce(f2.child_age, 0) = ?");
+    assertThat(loggedSql.get(0))
+      .contains("select t0.identifier from child_person t0")
+      .contains("left join (select i2.parent_identifier")
+      .contains("where coalesce(f2.child_age, 0) = ?");
   }
 
   @Test
@@ -232,7 +322,7 @@ public class TestQueryJoinOnFormula extends BaseTestCase {
 
     List<String> loggedSql = LoggedSqlCollector.stop();
     assertEquals(1, loggedSql.size());
-    assertThat(loggedSql.get(0)).contains("select count(*) from ( select t0.identifier");
+    assertThat(loggedSql.get(0)).contains("select count(*) from child_person t0 left join parent_person t1 on t1.identifier = t0.parent_identifier");
     assertThat(loggedSql.get(0)).contains("where coalesce(f2.child_age, 0) = ?");
   }
 }