mirror of
https://github.com/ebean-orm/ebean.git
synced 2026-11-18 14:58:47 +00:00
#405 - ENH: Add @ReadAudit and associated functionality for auditing reads including queries and L2 cache access
This commit is contained in:
1 parent
c292da736b
commit
54172aec6c
54 files changed
+2020
-200
No files matched your search
@@ -2,7 +2,14 @@ package com.avaje.ebeaninternal.server.query;
|
||||
|
||||
import com.avaje.ebean.QueryIterator;
|
||||
import com.avaje.ebean.Version;
|
||||
import com.avaje.ebean.bean.*;
|
||||
import com.avaje.ebean.bean.BeanCollection;
|
||||
import com.avaje.ebean.bean.EntityBean;
|
||||
import com.avaje.ebean.bean.EntityBeanIntercept;
|
||||
import com.avaje.ebean.bean.NodeUsageCollector;
|
||||
import com.avaje.ebean.bean.NodeUsageListener;
|
||||
import com.avaje.ebean.bean.ObjectGraphNode;
|
||||
import com.avaje.ebean.bean.PersistenceContext;
|
||||
import com.avaje.ebean.event.readaudit.ReadEvent;
|
||||
import com.avaje.ebeaninternal.api.SpiQuery;
|
||||
import com.avaje.ebeaninternal.api.SpiQuery.Mode;
|
||||
import com.avaje.ebeaninternal.api.SpiTransaction;
|
||||
@@ -10,7 +17,11 @@ import com.avaje.ebeaninternal.server.autofetch.AutoFetchManager;
|
||||
import com.avaje.ebeaninternal.server.core.Message;
|
||||
import com.avaje.ebeaninternal.server.core.OrmQueryRequest;
|
||||
import com.avaje.ebeaninternal.server.core.SpiOrmQueryRequest;
|
||||
import com.avaje.ebeaninternal.server.deploy.*;
|
||||
import com.avaje.ebeaninternal.server.deploy.BeanCollectionHelp;
|
||||
import com.avaje.ebeaninternal.server.deploy.BeanCollectionHelpFactory;
|
||||
import com.avaje.ebeaninternal.server.deploy.BeanDescriptor;
|
||||
import com.avaje.ebeaninternal.server.deploy.BeanPropertyAssocMany;
|
||||
import com.avaje.ebeaninternal.server.deploy.DbReadContext;
|
||||
import com.avaje.ebeaninternal.server.lib.util.StringHelper;
|
||||
import com.avaje.ebeaninternal.server.type.DataBind;
|
||||
import com.avaje.ebeaninternal.server.type.DataReader;
|
||||
@@ -24,6 +35,7 @@ import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
@@ -163,6 +175,16 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
|
||||
|
||||
private long executionTimeMicros;
|
||||
|
||||
/**
|
||||
* Flag set when findIterate is being read audited.
|
||||
*/
|
||||
private boolean auditFindIterate;
|
||||
|
||||
/**
|
||||
* A buffer of Ids collected for findIterate auditing.
|
||||
*/
|
||||
private List<Object> auditFindIterateIds;
|
||||
|
||||
/**
|
||||
* Create the Sql select based on the request.
|
||||
*/
|
||||
@@ -295,7 +317,7 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
|
||||
}
|
||||
|
||||
if (forwardOnlyHint) {
|
||||
// Use forward only hints for large resultset processing (Issue 56, MySql specific)
|
||||
// Use forward only hints for large resultSet processing (Issue 56, MySql specific)
|
||||
pstmt = conn.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
|
||||
pstmt.setFetchSize(Integer.MIN_VALUE);
|
||||
} else {
|
||||
@@ -327,18 +349,24 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
|
||||
/**
|
||||
* Close the resources.
|
||||
* <p>
|
||||
* The jdbc resultSet and statement need to be closed. Its important that this
|
||||
* method is called.
|
||||
* The JDBC resultSet and statement need to be closed. Its important that this method is called.
|
||||
* </p>
|
||||
*/
|
||||
public void close() {
|
||||
try {
|
||||
if (auditFindIterateIds != null && !auditFindIterateIds.isEmpty()) {
|
||||
auditIterateLogMessage();
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
logger.error("Error logging read audit logs", e);
|
||||
}
|
||||
try {
|
||||
if (dataReader != null) {
|
||||
dataReader.close();
|
||||
dataReader = null;
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
logger.error(null, e);
|
||||
logger.error("Error closing dataReader", e);
|
||||
}
|
||||
try {
|
||||
if (pstmt != null) {
|
||||
@@ -346,7 +374,7 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
|
||||
pstmt = null;
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
logger.error(null, e);
|
||||
logger.error("Error closing preparedStatement", e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -464,6 +492,9 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
|
||||
}
|
||||
|
||||
protected EntityBean next() {
|
||||
if (auditFindIterate) {
|
||||
auditIterateNextBean();
|
||||
}
|
||||
return nextBean;
|
||||
}
|
||||
|
||||
@@ -610,8 +641,7 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a PersistenceException including interesting information like the
|
||||
* bindLog and sql used.
|
||||
* Create a PersistenceException including interesting information like the bindLog and sql used.
|
||||
*/
|
||||
public PersistenceException createPersistenceException(SQLException e) {
|
||||
|
||||
@@ -619,8 +649,7 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a PersistenceException including interesting information like the
|
||||
* bindLog and sql used.
|
||||
* Create a PersistenceException including interesting information like the bindLog and sql used.
|
||||
*/
|
||||
public static PersistenceException createPersistenceException(SQLException e, SpiTransaction t, String bindLog, String sql) {
|
||||
|
||||
@@ -680,4 +709,71 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
|
||||
this.currentPathMap = currentPathMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* A find bean query with read auditing so build and log the ReadEvent.
|
||||
*/
|
||||
public void auditFind(EntityBean bean) {
|
||||
if (bean != null) {
|
||||
// only audit when a bean was actually found
|
||||
desc.readAuditBean(queryPlan.getAuditQueryKey(), bindLog, bean);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* a find many query with read auditing so build the ReadEvent and log it.
|
||||
*/
|
||||
public void auditFindMany() {
|
||||
|
||||
if (!collection.isEmpty()) {
|
||||
// get the id values of the underlying collection
|
||||
List<Object> ids = new ArrayList<Object>(collection.size());
|
||||
Collection<T> underlyingBeans = collection.getActualDetails();
|
||||
for (T underlyingBean : underlyingBeans) {
|
||||
ids.add(desc.getIdForJson(underlyingBean));
|
||||
}
|
||||
ReadEvent futureReadEvent = query.getFutureFetchAudit();
|
||||
if (futureReadEvent == null) {
|
||||
// normal query execution
|
||||
desc.readAuditMany(queryPlan.getAuditQueryKey(), bindLog, ids);
|
||||
} else {
|
||||
// this query was executed via findFutureList() and the prepare()
|
||||
// has already been called so set the details and log
|
||||
futureReadEvent.setQueryKey(queryPlan.getAuditQueryKey());
|
||||
futureReadEvent.setBindLog(bindLog);
|
||||
futureReadEvent.setIds(ids);
|
||||
desc.readAuditFutureMany(futureReadEvent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Indicate that read auditing is occurring on a this findIterate query.
|
||||
*/
|
||||
public void auditFindIterate() {
|
||||
auditFindIterate = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send the current buffer of findIterate collected ids to the audit log.
|
||||
*/
|
||||
private void auditIterateLogMessage() {
|
||||
desc.readAuditMany(queryPlan.getAuditQueryKey(), bindLog, auditFindIterateIds);
|
||||
// create a new list on demand with the next bean/id
|
||||
auditFindIterateIds = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add the id to the audit id buffer and flush if needed in batches of 100.
|
||||
*/
|
||||
private void auditIterateNextBean() {
|
||||
|
||||
if (auditFindIterateIds == null) {
|
||||
auditFindIterateIds = new ArrayList<Object>(100);
|
||||
}
|
||||
auditFindIterateIds.add(desc.getIdForJson(nextBean));
|
||||
if (auditFindIterateIds.size() >= 100) {
|
||||
auditIterateLogMessage();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import com.avaje.ebean.config.dbplatform.DatabasePlatform;
|
||||
import com.avaje.ebean.config.dbplatform.SqlLimitRequest;
|
||||
import com.avaje.ebean.config.dbplatform.SqlLimitResponse;
|
||||
import com.avaje.ebean.config.dbplatform.SqlLimiter;
|
||||
import com.avaje.ebean.event.readaudit.ReadAuditQueryPlan;
|
||||
import com.avaje.ebean.text.PathProperties;
|
||||
import com.avaje.ebeaninternal.api.ManyWhereJoins;
|
||||
import com.avaje.ebeaninternal.api.SpiQuery;
|
||||
@@ -269,6 +270,12 @@ public class CQueryBuilder {
|
||||
queryPlan = new CQueryPlan(request, res, sqlTree, false, predicates.getLogWhereSql());
|
||||
}
|
||||
|
||||
BeanDescriptor<T> desc = request.getBeanDescriptor();
|
||||
if (desc.isReadAuditing()) {
|
||||
// log the query plan based bean type (i.e. ignoring query disabling for logging the sql/plan)
|
||||
desc.getReadAuditLogger().queryPlan(new ReadAuditQueryPlan(desc.getFullName(), queryPlan.getAuditQueryKey(), queryPlan.getSql()));
|
||||
}
|
||||
|
||||
// cache the query plan because we can reuse it and also
|
||||
// gather query performance statistics based on it.
|
||||
request.putQueryPlan(queryPlan);
|
||||
|
||||
@@ -180,6 +180,11 @@ public class CQueryEngine {
|
||||
logFindManySummary(cquery);
|
||||
}
|
||||
|
||||
if (request.isAuditReads()) {
|
||||
// indicates we need to audit as the iterator progresses
|
||||
cquery.auditFindIterate();
|
||||
}
|
||||
|
||||
return readIterate;
|
||||
|
||||
} catch (SQLException e) {
|
||||
@@ -218,6 +223,10 @@ public class CQueryEngine {
|
||||
logFindManySummary(cquery);
|
||||
}
|
||||
|
||||
if (request.isAuditReads()) {
|
||||
cquery.auditFindMany();
|
||||
}
|
||||
|
||||
return versions;
|
||||
|
||||
} catch (SQLException e) {
|
||||
@@ -297,6 +306,10 @@ public class CQueryEngine {
|
||||
logFindManySummary(cquery);
|
||||
}
|
||||
|
||||
if (request.isAuditReads()) {
|
||||
cquery.auditFindMany();
|
||||
}
|
||||
|
||||
request.executeSecondaryQueries();
|
||||
|
||||
return beanCollection;
|
||||
@@ -342,6 +355,10 @@ public class CQueryEngine {
|
||||
logFindBeanSummary(cquery);
|
||||
}
|
||||
|
||||
if (request.isAuditReads()) {
|
||||
cquery.auditFind(bean);
|
||||
}
|
||||
|
||||
request.executeSecondaryQueries();
|
||||
|
||||
return (T) bean;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.avaje.ebeaninternal.server.query;
|
||||
|
||||
import java.security.MessageDigest;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
|
||||
@@ -14,6 +15,8 @@ import com.avaje.ebeaninternal.server.query.CQueryPlanStats.Snapshot;
|
||||
import com.avaje.ebeaninternal.server.type.DataBind;
|
||||
import com.avaje.ebeaninternal.server.type.DataReader;
|
||||
import com.avaje.ebeaninternal.server.type.RsetDataReader;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Represents a query for a given SQL statement.
|
||||
@@ -36,6 +39,8 @@ import com.avaje.ebeaninternal.server.type.RsetDataReader;
|
||||
*/
|
||||
public class CQueryPlan {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(CQueryPlan.class);
|
||||
|
||||
private final SpiEbeanServer server;
|
||||
|
||||
private final boolean autofetchTuned;
|
||||
@@ -61,6 +66,11 @@ public class CQueryPlan {
|
||||
|
||||
private final Class<?> beanType;
|
||||
|
||||
/**
|
||||
* Key used to identify the query plan in audit logging.
|
||||
*/
|
||||
private volatile String auditQueryHash;
|
||||
|
||||
/**
|
||||
* Create a query plan based on a OrmQueryRequest.
|
||||
*/
|
||||
@@ -141,6 +151,48 @@ public class CQueryPlan {
|
||||
return hash;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a key used in audit logging to identify the query.
|
||||
*/
|
||||
public String getAuditQueryKey() {
|
||||
if (auditQueryHash == null) {
|
||||
// volatile object assignment (so happy for multithreaded access)
|
||||
auditQueryHash = calcAuditQueryKey();
|
||||
}
|
||||
return auditQueryHash;
|
||||
}
|
||||
|
||||
private String calcAuditQueryKey() {
|
||||
// rawSql needs to include the MD5 hash of the sql
|
||||
return rawSql ? hash.getPartialKey() + "_" + getSqlMd5Hash() : hash.getPartialKey();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the MD5 hash of the underlying sql.
|
||||
*/
|
||||
private String getSqlMd5Hash() {
|
||||
try {
|
||||
MessageDigest md = MessageDigest.getInstance("MD5");
|
||||
byte[] digest = md.digest(sql.getBytes("UTF-8"));
|
||||
return digestToHex(digest);
|
||||
} catch (Exception e) {
|
||||
logger.error("Failed to MD5 hash the rawSql query", e);
|
||||
return "error";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert the digest into a hex value.
|
||||
*/
|
||||
private String digestToHex(byte[] digest) {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int i = 0; i < digest.length; i++) {
|
||||
sb.append(Integer.toString((digest[i] & 0xff) + 0x100, 16).substring(1));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public String getSql() {
|
||||
return sql;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user