#405 - ENH: Add @ReadAudit and associated functionality for auditing reads including queries and L2 cache access

This commit is contained in:
Robin Bygrave committed 2015-09-03 13:54:52 +12:00
1 parent c292da736b
commit 54172aec6c
54 files changed
+2020 -200

No files matched your search

@@ -2,7 +2,14 @@ package com.avaje.ebeaninternal.server.query;
import com.avaje.ebean.QueryIterator;
import com.avaje.ebean.Version;
import com.avaje.ebean.bean.*;
import com.avaje.ebean.bean.BeanCollection;
import com.avaje.ebean.bean.EntityBean;
import com.avaje.ebean.bean.EntityBeanIntercept;
import com.avaje.ebean.bean.NodeUsageCollector;
import com.avaje.ebean.bean.NodeUsageListener;
import com.avaje.ebean.bean.ObjectGraphNode;
import com.avaje.ebean.bean.PersistenceContext;
import com.avaje.ebean.event.readaudit.ReadEvent;
import com.avaje.ebeaninternal.api.SpiQuery;
import com.avaje.ebeaninternal.api.SpiQuery.Mode;
import com.avaje.ebeaninternal.api.SpiTransaction;
@@ -10,7 +17,11 @@ import com.avaje.ebeaninternal.server.autofetch.AutoFetchManager;
import com.avaje.ebeaninternal.server.core.Message;
import com.avaje.ebeaninternal.server.core.OrmQueryRequest;
import com.avaje.ebeaninternal.server.core.SpiOrmQueryRequest;
import com.avaje.ebeaninternal.server.deploy.*;
import com.avaje.ebeaninternal.server.deploy.BeanCollectionHelp;
import com.avaje.ebeaninternal.server.deploy.BeanCollectionHelpFactory;
import com.avaje.ebeaninternal.server.deploy.BeanDescriptor;
import com.avaje.ebeaninternal.server.deploy.BeanPropertyAssocMany;
import com.avaje.ebeaninternal.server.deploy.DbReadContext;
import com.avaje.ebeaninternal.server.lib.util.StringHelper;
import com.avaje.ebeaninternal.server.type.DataBind;
import com.avaje.ebeaninternal.server.type.DataReader;
@@ -24,6 +35,7 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -163,6 +175,16 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
private long executionTimeMicros;
/**
* Flag set when findIterate is being read audited.
*/
private boolean auditFindIterate;
/**
* A buffer of Ids collected for findIterate auditing.
*/
private List<Object> auditFindIterateIds;
/**
* Create the Sql select based on the request.
*/
@@ -295,7 +317,7 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
}
if (forwardOnlyHint) {
// Use forward only hints for large resultset processing (Issue 56, MySql specific)
// Use forward only hints for large resultSet processing (Issue 56, MySql specific)
pstmt = conn.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
pstmt.setFetchSize(Integer.MIN_VALUE);
} else {
@@ -327,18 +349,24 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
/**
* Close the resources.
* <p>
* The jdbc resultSet and statement need to be closed. Its important that this
* method is called.
* The JDBC resultSet and statement need to be closed. Its important that this method is called.
* </p>
*/
public void close() {
try {
if (auditFindIterateIds != null && !auditFindIterateIds.isEmpty()) {
auditIterateLogMessage();
}
} catch (Throwable e) {
logger.error("Error logging read audit logs", e);
}
try {
if (dataReader != null) {
dataReader.close();
dataReader = null;
}
} catch (SQLException e) {
logger.error(null, e);
logger.error("Error closing dataReader", e);
}
try {
if (pstmt != null) {
@@ -346,7 +374,7 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
pstmt = null;
}
} catch (SQLException e) {
logger.error(null, e);
logger.error("Error closing preparedStatement", e);
}
}
@@ -464,6 +492,9 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
}
protected EntityBean next() {
if (auditFindIterate) {
auditIterateNextBean();
}
return nextBean;
}
@@ -610,8 +641,7 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
}
/**
* Create a PersistenceException including interesting information like the
* bindLog and sql used.
* Create a PersistenceException including interesting information like the bindLog and sql used.
*/
public PersistenceException createPersistenceException(SQLException e) {
@@ -619,8 +649,7 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
}
/**
* Create a PersistenceException including interesting information like the
* bindLog and sql used.
* Create a PersistenceException including interesting information like the bindLog and sql used.
*/
public static PersistenceException createPersistenceException(SQLException e, SpiTransaction t, String bindLog, String sql) {
@@ -680,4 +709,71 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
this.currentPathMap = currentPathMap;
}
/**
* A find bean query with read auditing so build and log the ReadEvent.
*/
public void auditFind(EntityBean bean) {
if (bean != null) {
// only audit when a bean was actually found
desc.readAuditBean(queryPlan.getAuditQueryKey(), bindLog, bean);
}
}
/**
* a find many query with read auditing so build the ReadEvent and log it.
*/
public void auditFindMany() {
if (!collection.isEmpty()) {
// get the id values of the underlying collection
List<Object> ids = new ArrayList<Object>(collection.size());
Collection<T> underlyingBeans = collection.getActualDetails();
for (T underlyingBean : underlyingBeans) {
ids.add(desc.getIdForJson(underlyingBean));
}
ReadEvent futureReadEvent = query.getFutureFetchAudit();
if (futureReadEvent == null) {
// normal query execution
desc.readAuditMany(queryPlan.getAuditQueryKey(), bindLog, ids);
} else {
// this query was executed via findFutureList() and the prepare()
// has already been called so set the details and log
futureReadEvent.setQueryKey(queryPlan.getAuditQueryKey());
futureReadEvent.setBindLog(bindLog);
futureReadEvent.setIds(ids);
desc.readAuditFutureMany(futureReadEvent);
}
}
}
/**
* Indicate that read auditing is occurring on a this findIterate query.
*/
public void auditFindIterate() {
auditFindIterate = true;
}
/**
* Send the current buffer of findIterate collected ids to the audit log.
*/
private void auditIterateLogMessage() {
desc.readAuditMany(queryPlan.getAuditQueryKey(), bindLog, auditFindIterateIds);
// create a new list on demand with the next bean/id
auditFindIterateIds = null;
}
/**
* Add the id to the audit id buffer and flush if needed in batches of 100.
*/
private void auditIterateNextBean() {
if (auditFindIterateIds == null) {
auditFindIterateIds = new ArrayList<Object>(100);
}
auditFindIterateIds.add(desc.getIdForJson(nextBean));
if (auditFindIterateIds.size() >= 100) {
auditIterateLogMessage();
}
}
}
@@ -8,6 +8,7 @@ import com.avaje.ebean.config.dbplatform.DatabasePlatform;
import com.avaje.ebean.config.dbplatform.SqlLimitRequest;
import com.avaje.ebean.config.dbplatform.SqlLimitResponse;
import com.avaje.ebean.config.dbplatform.SqlLimiter;
import com.avaje.ebean.event.readaudit.ReadAuditQueryPlan;
import com.avaje.ebean.text.PathProperties;
import com.avaje.ebeaninternal.api.ManyWhereJoins;
import com.avaje.ebeaninternal.api.SpiQuery;
@@ -269,6 +270,12 @@ public class CQueryBuilder {
queryPlan = new CQueryPlan(request, res, sqlTree, false, predicates.getLogWhereSql());
}
BeanDescriptor<T> desc = request.getBeanDescriptor();
if (desc.isReadAuditing()) {
// log the query plan based bean type (i.e. ignoring query disabling for logging the sql/plan)
desc.getReadAuditLogger().queryPlan(new ReadAuditQueryPlan(desc.getFullName(), queryPlan.getAuditQueryKey(), queryPlan.getSql()));
}
// cache the query plan because we can reuse it and also
// gather query performance statistics based on it.
request.putQueryPlan(queryPlan);
@@ -180,6 +180,11 @@ public class CQueryEngine {
logFindManySummary(cquery);
}
if (request.isAuditReads()) {
// indicates we need to audit as the iterator progresses
cquery.auditFindIterate();
}
return readIterate;
} catch (SQLException e) {
@@ -218,6 +223,10 @@ public class CQueryEngine {
logFindManySummary(cquery);
}
if (request.isAuditReads()) {
cquery.auditFindMany();
}
return versions;
} catch (SQLException e) {
@@ -297,6 +306,10 @@ public class CQueryEngine {
logFindManySummary(cquery);
}
if (request.isAuditReads()) {
cquery.auditFindMany();
}
request.executeSecondaryQueries();
return beanCollection;
@@ -342,6 +355,10 @@ public class CQueryEngine {
logFindBeanSummary(cquery);
}
if (request.isAuditReads()) {
cquery.auditFind(bean);
}
request.executeSecondaryQueries();
return (T) bean;
@@ -1,5 +1,6 @@
package com.avaje.ebeaninternal.server.query;
import java.security.MessageDigest;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -14,6 +15,8 @@ import com.avaje.ebeaninternal.server.query.CQueryPlanStats.Snapshot;
import com.avaje.ebeaninternal.server.type.DataBind;
import com.avaje.ebeaninternal.server.type.DataReader;
import com.avaje.ebeaninternal.server.type.RsetDataReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Represents a query for a given SQL statement.
@@ -36,6 +39,8 @@ import com.avaje.ebeaninternal.server.type.RsetDataReader;
*/
public class CQueryPlan {
private static final Logger logger = LoggerFactory.getLogger(CQueryPlan.class);
private final SpiEbeanServer server;
private final boolean autofetchTuned;
@@ -61,6 +66,11 @@ public class CQueryPlan {
private final Class<?> beanType;
/**
* Key used to identify the query plan in audit logging.
*/
private volatile String auditQueryHash;
/**
* Create a query plan based on a OrmQueryRequest.
*/
@@ -141,6 +151,48 @@ public class CQueryPlan {
return hash;
}
/**
* Return a key used in audit logging to identify the query.
*/
public String getAuditQueryKey() {
if (auditQueryHash == null) {
// volatile object assignment (so happy for multithreaded access)
auditQueryHash = calcAuditQueryKey();
}
return auditQueryHash;
}
private String calcAuditQueryKey() {
// rawSql needs to include the MD5 hash of the sql
return rawSql ? hash.getPartialKey() + "_" + getSqlMd5Hash() : hash.getPartialKey();
}
/**
* Return the MD5 hash of the underlying sql.
*/
private String getSqlMd5Hash() {
try {
MessageDigest md = MessageDigest.getInstance("MD5");
byte[] digest = md.digest(sql.getBytes("UTF-8"));
return digestToHex(digest);
} catch (Exception e) {
logger.error("Failed to MD5 hash the rawSql query", e);
return "error";
}
}
/**
* Convert the digest into a hex value.
*/
private String digestToHex(byte[] digest) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < digest.length; i++) {
sb.append(Integer.toString((digest[i] & 0xff) + 0x100, 16).substring(1));
}
return sb.toString();
}
public String getSql() {
return sql;
}