JSON bean dirty detection via MD5 of JSON string content

- MD5 of json content stored on EntityBeanIntercept for dirty detection
- Only convert to JSON once (at dirty detection time). Store this json content on EntityBeanIntercept to later push to ScalarTypeJsonObjectMapper for bind

Should consider alternative to extend BeanProperty rather than have these if blocks.

ebean-api/src/main/java/io/ebean/bean/EntityBeanIntercept.java

@@ -91,6 +91,17 @@ public final class EntityBeanIntercept implements Serializable {
   private Object ownerId;
   private int sortOrder;
 
+  /**
+   * Holds MD5 hash of json loaded jackson beans.
+   */
+  private String[] mutableHash;
+
+  /**
+   * Holds json content determined at point of dirty check.
+   * Stored here on dirty check such that we only convert to json once.
+   */
+  private String[] mutableContent;
+
   /**
    * Create a intercept with a given entity.
    */
@@ -1138,4 +1149,26 @@ public final class EntityBeanIntercept implements Serializable {
     }
     return ret;
   }
+
+  public String mutableHash(int propertyIndex) {
+    return mutableHash == null ? null : mutableHash[propertyIndex];
+  }
+
+  public void mutableHash(int propertyIndex, String content) {
+    if (mutableHash == null) {
+      mutableHash = new String[flags.length];
+    }
+    mutableHash[propertyIndex] = content;
+  }
+
+  public String mutableContent(int propertyIndex) {
+    return mutableContent == null ? null : mutableContent[propertyIndex];
+  }
+
+  public void mutableContent(int propertyIndex, String content) {
+    if (mutableContent == null) {
+      mutableContent = new String[flags.length];
+    }
+    mutableContent[propertyIndex] = content;
+  }
 }

ebean-core-type/src/main/java/io/ebean/core/type/DataBinder.java

@@ -163,4 +163,15 @@ public interface DataBinder {
    * Bind an array value.
    */
   void setArray(String arrayType, Object[] elements) throws SQLException;
+
+  /**
+   * Push json from dirty detection to be available for binding.
+   */
+  void pushJson(String json);
+
+  /**
+   * Pop json made during dirty detection for scalarType binding.
+   */
+  String popJson();
+
 }

ebean-core-type/src/main/java/io/ebean/core/type/DataReader.java

@@ -48,4 +48,14 @@ public interface DataReader {
   Object getObject() throws SQLException;
 
   InputStream getBinaryStream() throws SQLException;
+
+  /**
+   * Push json from dirty detection to be available for binding.
+   */
+  void pushJson(String json);
+
+  /**
+   * Pop json made during dirty detection for scalarType binding.
+   */
+  String popJson();
 }

ebean-core-type/src/main/java/io/ebean/core/type/ScalarType.java

@@ -34,6 +34,14 @@ import java.sql.SQLException;
  */
 public interface ScalarType<T> extends StringParser, StringFormatter, ScalarDataReader<T> {
 
+  default boolean isJsonMapper() {
+    return false;
+  }
+
+  default String jsonMapper(Object value) {
+    throw new UnsupportedOperationException();
+  }
+
   /**
    * Return true if this is a binary type and can not support parse() and format() from/to string.
    * This allows Ebean to optimise marshalling types to string.

ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanDescriptor.java

@@ -2022,7 +2022,7 @@ public class BeanDescriptor<T> implements BeanType<T>, STreeType, SpiBeanType {
   public boolean isTableManaged(String tableName) {
     return owner.isTableManaged(tableName);
   }
-  
+
   /**
    * Return the order column property.
    */
@@ -3200,7 +3200,7 @@ public class BeanDescriptor<T> implements BeanType<T>, STreeType, SpiBeanType {
       int propertyIndex = beanProperty.getPropertyIndex();
       if (!ebi.isDirtyProperty(propertyIndex) && ebi.isLoadedProperty(propertyIndex)) {
         Object value = beanProperty.getValue(ebi.getOwner());
-        if (value != null && beanProperty.isDirtyValue(value)) {
+        if (value != null && beanProperty.isDirtyValue(value, ebi)) {
           // mutable scalar value which is considered dirty so mark
           // it as such so that it is included in an update
           ebi.markPropertyAsChanged(propertyIndex);

ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanProperty.java

@@ -13,6 +13,7 @@ import io.ebean.core.type.DocPropertyType;
 import io.ebean.core.type.ScalarType;
 import io.ebean.plugin.Property;
 import io.ebean.text.StringParser;
+import io.ebean.text.TextException;
 import io.ebean.util.SplitName;
 import io.ebeaninternal.api.SpiExpressionRequest;
 import io.ebeaninternal.api.SpiQuery;
@@ -31,11 +32,8 @@ import io.ebeaninternal.server.properties.BeanPropertySetter;
 import io.ebeaninternal.server.query.STreeProperty;
 import io.ebeaninternal.server.query.SqlBeanLoad;
 import io.ebeaninternal.server.query.SqlJoinType;
-import io.ebeaninternal.server.type.DataBind;
-import io.ebeaninternal.server.type.LocalEncryptedType;
-import io.ebeaninternal.server.type.ScalarTypeBoolean;
-import io.ebeaninternal.server.type.ScalarTypeEnum;
-import io.ebeaninternal.server.type.ScalarTypeLogicalType;
+import io.ebeaninternal.server.type.*;
+import io.ebeaninternal.server.util.Md5;
 import io.ebeaninternal.util.ValueUtil;
 import io.ebeanservice.docstore.api.mapping.DocMappingBuilder;
 import io.ebeanservice.docstore.api.mapping.DocPropertyMapping;
@@ -54,6 +52,7 @@ import java.sql.SQLException;
 import java.sql.Types;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 /**
@@ -220,6 +219,7 @@ public class BeanProperty implements ElPropertyValue, Property, STreeProperty {
    */
   @SuppressWarnings("rawtypes")
   final ScalarType scalarType;
+  final boolean jsonMapperType;
 
   private final DocPropertyOptions docOptions;
 
@@ -333,6 +333,7 @@ public class BeanProperty implements ElPropertyValue, Property, STreeProperty {
     this.formula = sqlFormulaSelect != null;
     this.dbType = deploy.getDbType();
     this.scalarType = deploy.getScalarType();
+    this.jsonMapperType = (scalarType == null) ? false : scalarType.isJsonMapper();
     this.lob = isLobType(dbType);
     this.propertyType = deploy.getPropertyType();
     this.field = deploy.getField();
@@ -427,6 +428,7 @@ public class BeanProperty implements ElPropertyValue, Property, STreeProperty {
     this.setter = source.setter;
     this.dbType = source.getDbType(true);
     this.scalarType = source.scalarType;
+    this.jsonMapperType = source.jsonMapperType;
     this.lob = isLobType(dbType);
     this.propertyType = source.getPropertyType();
     this.field = source.getField();
@@ -630,8 +632,17 @@ public class BeanProperty implements ElPropertyValue, Property, STreeProperty {
       Object value = scalarType.read(reader);
       if (bean != null) {
         setValue(bean, value);
+        if (jsonMapperType) {
+          String json = reader.popJson();
+          if (json != null) {
+            final String hash = Md5.hash(json);
+            bean._ebean_getIntercept().mutableHash(propertyIndex, hash);
+          }
+        }
       }
       return value;
+    } catch (TextException e) {
+      throw e;
     } catch (Exception e) {
       throw new PersistenceException("Error readSet on " + descriptor + "." + name, e);
     }
@@ -643,13 +654,11 @@ public class BeanProperty implements ElPropertyValue, Property, STreeProperty {
 
   public Object readSet(DbReadContext ctx, EntityBean bean) throws SQLException {
     try {
-      Object value = scalarType.read(ctx.getDataReader());
-      if (bean != null) {
-        setValue(bean, value);
-      }
-      return value;
-    } catch (Exception e) {
-      throw new PersistenceException("Error readSet on " + descriptor + "." + name, e);
+      return readSet(ctx.getDataReader(), bean);
+    } catch (TextException e) {
+      bean._ebean_getIntercept().setLoadError(propertyIndex, e);
+      ctx.handleLoadError(getFullBeanName(), e);
+      return getValue(bean);
     }
   }
 
@@ -1018,7 +1027,19 @@ public class BeanProperty implements ElPropertyValue, Property, STreeProperty {
    * Return true if the mutable value is considered dirty.
    * This is only used for 'mutable' scalar types like hstore etc.
    */
-  boolean isDirtyValue(Object value) {
+  boolean isDirtyValue(Object value, EntityBeanIntercept ebi) {
+    if (jsonMapperType) {
+      // dirty detection based on md5 hash of json content
+      final String json = scalarType.jsonMapper(value);
+      final String newHash = Md5.hash(json);
+      final String oldHash = ebi.mutableHash(propertyIndex);
+      if (!Objects.equals(newHash, oldHash)) {
+        ebi.mutableContent(propertyIndex, json); // so we only convert to json once
+        ebi.mutableHash(propertyIndex, newHash); // for dirty detection next time
+        return true;
+      }
+      return false;
+    }
     return scalarType.isDirty(value);
   }
 

ebean-core/src/main/java/io/ebeaninternal/server/persist/dml/DmlHandler.java

@@ -63,6 +63,11 @@ public abstract class DmlHandler implements PersistHandler, BindableRequest {
     }
   }
 
+  @Override
+  public void pushJson(String json) {
+    dataBind.pushJson(json);
+  }
+
   @Override
   public long now() {
     return now;

ebean-core/src/main/java/io/ebeaninternal/server/persist/dmlbind/BindablePropertyJsonInsert.java

@@ -0,0 +1,42 @@
+package io.ebeaninternal.server.persist.dmlbind;
+
+import io.ebean.bean.EntityBean;
+import io.ebeaninternal.server.deploy.BeanProperty;
+import io.ebeaninternal.server.util.Md5;
+
+import java.sql.SQLException;
+
+/**
+ * For JSON Jackson properties - dirty detection via MD5 of json content.
+ */
+class BindablePropertyJsonInsert extends BindableProperty {
+
+  private final int propertyIndex;
+
+  BindablePropertyJsonInsert(BeanProperty prop) {
+    super(prop);
+    this.propertyIndex = prop.getPropertyIndex();
+  }
+
+  /**
+   * Normal binding of a property value from the bean.
+   */
+  @Override
+  public void dmlBind(BindableRequest request, EntityBean bean) throws SQLException {
+    if (bean == null) {
+      request.bind(null, prop);
+    } else {
+      Object value = prop.getValue(bean);
+      if (value == null) {
+        request.bind(null, prop);
+      } else {
+        // on insert store MD5 hash and push json
+        final String json = prop.format(value);
+        final String hash = Md5.hash(json);
+        bean._ebean_getIntercept().mutableHash(propertyIndex, hash);
+        request.pushJson(json);
+        request.bind(value, prop);
+      }
+    }
+  }
+}

ebean-core/src/main/java/io/ebeaninternal/server/persist/dmlbind/BindablePropertyJsonUpdate.java

@@ -0,0 +1,35 @@
+package io.ebeaninternal.server.persist.dmlbind;
+
+import io.ebean.bean.EntityBean;
+import io.ebeaninternal.server.deploy.BeanProperty;
+
+import java.sql.SQLException;
+
+/**
+ * For JSON Jackson properties - dirty detection via MD5 of json content.
+ */
+class BindablePropertyJsonUpdate extends BindableProperty {
+
+  private final int propertyIndex;
+
+  BindablePropertyJsonUpdate(BeanProperty prop) {
+    super(prop);
+    this.propertyIndex = prop.getPropertyIndex();
+  }
+
+  /**
+   * Normal binding of a property value from the bean.
+   */
+  @Override
+  public void dmlBind(BindableRequest request, EntityBean bean) throws SQLException {
+    if (bean == null) {
+      request.bind(null, prop);
+    } else {
+      // on update push json
+      final String json = bean._ebean_getIntercept().mutableContent(propertyIndex);
+      request.pushJson(json);
+      final Object value = prop.getValue(bean);
+      request.bind(value, prop);
+    }
+  }
+}

ebean-core/src/main/java/io/ebeaninternal/server/persist/dmlbind/BindableRequest.java

@@ -57,4 +57,9 @@ public interface BindableRequest {
    * Return true if this is an update request.
    */
   boolean isUpdate();
+
+  /**
+   * Push json content for scalarType bind().
+   */
+  void pushJson(String json);
 }

ebean-core/src/main/java/io/ebeaninternal/server/persist/dmlbind/FactoryProperty.java

@@ -43,6 +43,13 @@ class FactoryProperty {
       return  new BindableAssocOne((BeanPropertyAssocOne<?>)prop);
     }
 
+    if (prop.getScalarType().isJsonMapper()) {
+      if (DmlMode.INSERT == mode) {
+        return new BindablePropertyJsonInsert(prop);
+      } else if (DmlMode.UPDATE == mode) {
+        return new BindablePropertyJsonUpdate(prop);
+      }
+    }
     return new BindableProperty(prop);
   }
 }

ebean-core/src/main/java/io/ebeaninternal/server/query/SqlBeanLoad.java

@@ -25,7 +25,6 @@ public class SqlBeanLoad {
   private final boolean rawSql;
 
   SqlBeanLoad(DbReadContext ctx, Class<?> type, EntityBean bean, Mode queryMode) {
-
     this.ctx = ctx;
     this.rawSql = ctx.isRawSql();
     this.type = type;
@@ -69,16 +68,16 @@ public class SqlBeanLoad {
     }
 
     try {
-      Object dbVal = prop.read(ctx);
       if (!refreshLoading) {
-        prop.setValue(bean, dbVal);
-      } else {
-        prop.setValueIntercept(bean, dbVal);
+        return prop.readSet(ctx, bean);
       }
-
+      // TODO: maybe create prop.readSetIntercept() and move this
+      Object dbVal = prop.read(ctx);
+      prop.setValueIntercept(bean, dbVal);
       return dbVal;
 
     } catch (Exception e) {
+      // TODO: maybe move this into prop.readSetIntercept()
       bean._ebean_getIntercept().setLoadError(prop.getPropertyIndex(), e);
       ctx.handleLoadError(prop.getFullBeanName(), e);
       return prop.getValue(bean);

ebean-core/src/main/java/io/ebeaninternal/server/type/DataBind.java

@@ -36,6 +36,7 @@ public class DataBind implements DataBinder {
   private List<InputStream> inputStreams;
 
   protected int pos;
+  private String json;
 
   public DataBind(DataTimeZone dataTimeZone, PreparedStatement pstmt, Connection connection) {
     this.dataTimeZone = dataTimeZone;
@@ -43,6 +44,16 @@ public class DataBind implements DataBinder {
     this.connection = connection;
   }
 
+  @Override
+  public void pushJson(String json) {
+    this.json = json;
+  }
+
+  @Override
+  public String popJson() {
+    return json;
+  }
+
   @Override
   public StringBuilder append(Object entry) {
     return bindLog.append(entry);

ebean-core/src/main/java/io/ebeaninternal/server/type/RsetDataReader.java

@@ -20,22 +20,29 @@ import java.util.Calendar;
 public class RsetDataReader implements DataReader {
 
   private static final int bufferSize = 512;
-
   static final int clobBufferSize = 512;
-
   static final int stringInitialSize = 512;
 
   private final DataTimeZone dataTimeZone;
-
   private final ResultSet rset;
-
   protected int pos;
+  private String json;
 
   public RsetDataReader(DataTimeZone dataTimeZone, ResultSet rset) {
     this.dataTimeZone = dataTimeZone;
     this.rset = rset;
   }
 
+  @Override
+  public void pushJson(String json) {
+    this.json = json;
+  }
+
+  @Override
+  public String popJson() {
+    return json;
+  }
+
   @Override
   public void close() throws SQLException {
     rset.close();

ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypeJsonObjectMapper.java

@@ -56,6 +56,50 @@ class ScalarTypeJsonObjectMapper {
     GenericObject(TypeJsonManager jsonManager, AnnotatedField field, int dbType, Class<?> rawType) {
       super(Object.class, jsonManager, field, dbType, DocPropertyType.OBJECT, rawType);
     }
+
+    @Override
+    public boolean isJsonMapper() {
+      return true;
+    }
+
+    @Override
+    public String jsonMapper(Object value) {
+      return formatValue(value);
+    }
+
+    @Override
+    public Object read(DataReader reader) throws SQLException {
+      String json = reader.getString();
+      if (json == null || json.isEmpty()) {
+        return null;
+      }
+      // pushJson such that we MD5 and store on EntityBeanIntercept later
+      reader.pushJson(json);
+      try {
+        return objectReader.readValue(json, deserType);
+      } catch (IOException e) {
+        throw new TextException("Failed to parse JSON [{}] as " + deserType, json, e);
+      }
+    }
+
+    @Override
+    public void bind(DataBinder binder, Object value) throws SQLException {
+      // popJson as dirty detection already converted to json string
+      String rawJson = binder.popJson();
+      if (rawJson == null && value != null) {
+        rawJson = formatValue(value); // not expected, need to check?
+      }
+      if (pgType != null) {
+        binder.setObject(PostgresHelper.asObject(pgType, rawJson));
+      } else {
+        if (value == null) {
+          // use varchar, otherwise SqlServer/db2 will fail with 'Invalid JDBC data type 5.001.'
+          binder.setNull(Types.VARCHAR);
+        } else {
+          binder.setString(rawJson);
+        }
+      }
+    }
   }
 
   /**
@@ -118,10 +162,10 @@ class ScalarTypeJsonObjectMapper {
    */
   private static abstract class Base<T> extends ScalarTypeBase<T> {
 
-    private final ObjectWriter objectWriter;
-    private final ObjectMapper objectReader;
-    private final JavaType deserType;
-    private final String pgType;
+    protected final ObjectWriter objectWriter;
+    protected final ObjectMapper objectReader;
+    protected final JavaType deserType;
+    protected final String pgType;
     private final DocPropertyType docType;
     private final TypeJsonManager.DirtyHandler dirtyHandler;
 

ebean-core/src/test/java/org/tests/json/TestDbJson_Jackson3.java

@@ -74,6 +74,7 @@ public class TestDbJson_Jackson3 extends BaseTestCase {
 
     final List<String> sql = LoggedSql.stop();
     assertThat(sql).hasSize(1);
-    assertThat(sql.get(0)).contains("update ebasic_json_list set name=?, bean_list=?, plain_bean=?, version=? where id=?");
+    // plain_bean=?, no longer included with MD5 dirty detection
+    assertThat(sql.get(0)).contains("update ebasic_json_list set name=?, bean_list=?, version=? where id=?");
   }
 }

ebean-core/src/test/java/org/tests/json/TestDbJson_List.java

@@ -113,7 +113,8 @@ public class TestDbJson_List extends BaseTestCase {
     List<String> sql = LoggedSqlCollector.stop();
 
     // we don't update the phone numbers (as they are not dirty)
-    assertSql(sql.get(0)).contains("update ebasic_json_list set name=?, plain_bean=?, version=? where");
+    // plain_bean=?, no longer included with MD5 dirty detection
+    assertSql(sql.get(0)).contains("update ebasic_json_list set name=?, version=? where");
   }
 
   public void update_when_dirty() {
@@ -126,7 +127,8 @@ public class TestDbJson_List extends BaseTestCase {
     List<String> sql = LoggedSqlCollector.stop();
 
     // we don't update the phone numbers (as they are not dirty)
-    assertSql(sql.get(0)).contains("update ebasic_json_list set plain_bean=?, tags=?, version=? where id=? and version=?");
+    // plain_bean=? not included using MD5 dirty detection
+    assertSql(sql.get(0)).contains("update ebasic_json_list set tags=?, version=? where id=? and version=?");
   }
 
   public void update_when_dirty_flags() {
@@ -139,7 +141,8 @@ public class TestDbJson_List extends BaseTestCase {
     List<String> sql = LoggedSqlCollector.stop();
 
     // we don't update the phone numbers (as they are not dirty)
-    assertSql(sql.get(0)).contains("update ebasic_json_list set plain_bean=?, flags=?, version=? where id=? and version=?;");
+    // plain_bean=? not included with MD5 dirty detection
+    assertSql(sql.get(0)).contains("update ebasic_json_list set flags=?, version=? where id=? and version=?;");
   }
 
   public void update_when_dirty_SetListMap() {
@@ -154,7 +157,8 @@ public class TestDbJson_List extends BaseTestCase {
     List<String> sql = LoggedSqlCollector.stop();
 
     // we don't update the phone numbers (as they are not dirty)
-    assertSql(sql.get(0)).contains("update ebasic_json_list set beans=?, bean_list=?, bean_map=?, plain_bean=?, version=? where id=? and version=?");
+    // plain_bean=? not included with MD5 dirty detection
+    assertSql(sql.get(0)).contains("update ebasic_json_list set beans=?, bean_list=?, bean_map=?, version=? where id=? and version=?");
   }
 
   @Test

ebean-core/src/test/java/org/tests/model/json/EBasicPlain.java

@@ -0,0 +1,54 @@
+package org.tests.model.json;
+
+import io.ebean.annotation.DbJson;
+
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Version;
+
+@Entity
+public class EBasicPlain {
+
+  @Id
+  long id;
+
+  String attr;
+
+  @DbJson(length = 500)
+  PlainBean plainBean;
+
+  @Version
+  long version;
+
+  public long getId() {
+    return id;
+  }
+
+  public void setId(long id) {
+    this.id = id;
+  }
+
+  public String getAttr() {
+    return attr;
+  }
+
+  public void setAttr(String attr) {
+    this.attr = attr;
+  }
+
+  public PlainBean getPlainBean() {
+    return plainBean;
+  }
+
+  public void setPlainBean(PlainBean plainBean) {
+    this.plainBean = plainBean;
+  }
+
+  public long getVersion() {
+    return version;
+  }
+
+  public void setVersion(long version) {
+    this.version = version;
+  }
+}

ebean-core/src/test/java/org/tests/model/json/TestJacksonPlainBean.java

@@ -0,0 +1,73 @@
+package org.tests.model.json;
+
+import io.ebean.DB;
+import org.ebeantest.LoggedSqlCollector;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class TestJacksonPlainBean {
+
+  @Test
+  public void insertUpdate() {
+
+    DB.getDefault();
+    LoggedSqlCollector.start();
+
+    PlainBean content = new PlainBean();
+    content.setAlong(42);
+    content.setName("foo");
+
+    EBasicPlain bean = new EBasicPlain();
+    bean.setAttr("attr0");
+    bean.setPlainBean(content);
+
+
+    DB.save(bean);
+    expectedSql(0, "insert into ebasic_plain (attr, plain_bean, version) values (?,?,?)");
+
+
+    // inserted plainBean has not been mutated
+    bean.setAttr("attr1");
+    DB.save(bean);
+    expectedSql(0, "update ebasic_plain set attr=?, version=? where id=? and version=?");
+
+
+    // inserted plainBean has now been mutated
+    content.setName("notFoo");
+    bean.setAttr("attr2");
+    DB.save(bean);
+    expectedSql(0, "update ebasic_plain set attr=?, plain_bean=?, version=? where id=? and version=?");
+
+
+    final EBasicPlain found = DB.find(EBasicPlain.class, bean.getId());
+
+    // update mutating PlainBean only
+    final PlainBean plainBean = found.getPlainBean();
+    plainBean.setName("mod1");
+    DB.save(found);
+    expectedSql(1, "update ebasic_plain set plain_bean=?, version=? where id=? and version=?");
+
+
+    // update bean, mutate PlainBean only
+    plainBean.setName("mod2");
+    DB.save(found);
+    expectedSql(0, "update ebasic_plain set plain_bean=?, version=? where id=? and version=?");
+
+
+    // update bean, not mutating PlainBean
+    found.setAttr("attr3");
+    DB.save(found);
+    expectedSql(LoggedSqlCollector.stop(), 0, "update ebasic_plain set attr=?, version=? where id=? and version=?");
+  }
+
+  private void expectedSql(int i, String s) {
+    assertThat(LoggedSqlCollector.current().get(i)).contains(s);
+  }
+
+  private void expectedSql(List<String> sql, int i, String s) {
+    assertThat(sql.get(i)).contains(s);
+  }
+}