1476 - Query plan capture, initial wip capturing bind params (#1477)

* 1476 - Query plan capture, initial wip capturing bind params

* No effective change - Reset test default db to H2

* #1476 - Query plan capture, initial wip capturing bind params #1477

Update with consumer

* #1476 - Query plan capture, initial wip capturing bind params #1477

Update with query plan request

* #1476 - Query plan capture, initial wip capturing bind params #1477

Extract interface

* #1476 - Query plan capture, initial wip capturing bind params #1477

Extract interface
This commit is contained in:
Rob Bygrave authored and GitHub committed 2018-12-19 16:04:28 +13:00
1 parent e611353163
commit 94591954b3
108 files changed
+1908 -233

No files matched your search

@@ -602,7 +602,9 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
if (autoTuneProfiling) {
profilingListener.collectQueryInfo(objectGraphNode, loadedBeanCount, executionTimeMicros);
}
queryPlan.executionTime(loadedBeanCount, executionTimeMicros, objectGraphNode);
if (queryPlan.executionTime(loadedBeanCount, executionTimeMicros, objectGraphNode)) {
queryPlan.captureBindForQueryPlan(predicates, executionTimeMicros);
}
getTransaction().profileEvent(this);
} catch (Exception e) {
logger.error("Error updating execution statistics", e);
@@ -0,0 +1,73 @@
package io.ebeaninternal.server.query;
import io.ebean.config.ServerConfig;
import io.ebean.meta.QueryPlanRequest;
import io.ebeaninternal.server.type.bindcapture.BindCapture;
class CQueryBindCapture {
private final double multiplier = 1.3d;
private final CQueryPlan cQueryPlan;
private final QueryPlanLogger planLogger;
private final boolean enabled;
private BindCapture bindCapture;
private long queryTimeMicros;
private long thresholdMicros;
private long captureCount;
private long lastBindCapture;
CQueryBindCapture(CQueryPlan cQueryPlan, ServerConfig serverConfig) {
this.cQueryPlan = cQueryPlan;
this.enabled = serverConfig.isCollectQueryPlans();
this.planLogger = PlatformQueryPlan.getLogger(serverConfig.getDatabasePlatform().getPlatform());
}
/**
* Return true if we should capture the bind values for this query.
*/
boolean collectFor(long timeMicros) {
return enabled && (bindCapture == null || timeMicros > thresholdMicros);
}
/**
* Set the captured bind values that we can use later to collect a query plan.
*
* @param bindCapture The bind values of the query
* @param queryTimeMicros The query execution time
*/
void setBind(BindCapture bindCapture, long queryTimeMicros) {
synchronized (this) {
this.bindCapture = bindCapture;
this.queryTimeMicros = queryTimeMicros;
this.thresholdMicros = Math.round(queryTimeMicros * multiplier);
captureCount++;
lastBindCapture = System.currentTimeMillis();
}
}
/**
* Collect the query plan using already captured bind values.
*/
void collectQueryPlan(QueryPlanRequest request) {
if (request.getSince() > lastBindCapture) {
// no bind capture since the last capture
return;
}
final BindCapture last = this.bindCapture;
if (last == null) {
return;
}
DQueryPlanOutput queryPlan = planLogger.logQueryPlan(request.getConnection(), cQueryPlan, last);
queryPlan.with(queryTimeMicros, captureCount, cQueryPlan.getPlanKey().toString());
request.process(queryPlan);
}
}
@@ -28,7 +28,7 @@ class CQueryBuilderRawSql {
if (rsql == null) {
// this is a ResultSet based RawSql query - just use some placeholder for the SQL
return new SqlLimitResponse("--ResultSetBasedRawSql", false);
return new SqlLimitResponse(CQueryPlan.RESULT_SET_BASED_RAW_SQL, false);
}
if (!rsql.isParsed()) {
@@ -117,7 +117,9 @@ class CQueryFetchSingleAttribute implements SpiProfileTransactionEvent {
executionTimeMicros = (System.nanoTime() - startNano) / 1000L;
request.slowQueryCheck(executionTimeMicros, rowCount);
queryPlan.executionTime(rowCount, executionTimeMicros, null);
if (queryPlan.executionTime(rowCount, executionTimeMicros, null)) {
queryPlan.captureBindForQueryPlan(predicates, executionTimeMicros);
}
getTransaction().profileEvent(this);
return result;
@@ -4,6 +4,7 @@ import io.ebean.ProfileLocation;
import io.ebean.bean.ObjectGraphNode;
import io.ebean.config.dbplatform.SqlLimitResponse;
import io.ebean.meta.MetricType;
import io.ebean.meta.QueryPlanRequest;
import io.ebean.metric.MetricFactory;
import io.ebean.metric.TimedMetric;
import io.ebeaninternal.api.CQueryPlanKey;
@@ -13,6 +14,7 @@ import io.ebeaninternal.server.core.OrmQueryRequest;
import io.ebeaninternal.server.core.timezone.DataTimeZone;
import io.ebeaninternal.server.query.CQueryPlanStats.Snapshot;
import io.ebeaninternal.server.type.DataBind;
import io.ebeaninternal.server.type.DataBindCapture;
import io.ebeaninternal.server.type.DataReader;
import io.ebeaninternal.server.type.RsetDataReader;
import io.ebeaninternal.server.type.ScalarDataReader;
@@ -50,6 +52,8 @@ public class CQueryPlan {
private static final Logger logger = LoggerFactory.getLogger(CQueryPlan.class);
public static final String RESULT_SET_BASED_RAW_SQL = "--ResultSetBasedRawSql";
private final SpiEbeanServer server;
private final boolean autoTuned;
@@ -92,6 +96,8 @@ public class CQueryPlan {
private final Set<String> dependentTables;
private final CQueryBindCapture bindCapture;
/**
* Create a query plan based on a OrmQueryRequest.
*/
@@ -115,6 +121,7 @@ public class CQueryPlan {
this.encryptedProps = sqlTree.getEncryptedProps();
this.stats = new CQueryPlanStats(this, server.isCollectQueryOrigins());
this.dependentTables = sqlTree.dependentTables();
this.bindCapture = new CQueryBindCapture(this, server.getServerConfig());
}
/**
@@ -140,6 +147,7 @@ public class CQueryPlan {
this.encryptedProps = sqlTree.getEncryptedProps();
this.stats = new CQueryPlanStats(this, server.isCollectQueryOrigins());
this.dependentTables = (rawSql) ? Collections.emptySet() : sqlTree.dependentTables();
this.bindCapture = new CQueryBindCapture(this, server.getServerConfig());
}
private String location() {
@@ -193,6 +201,16 @@ public class CQueryPlan {
return dataBind;
}
private DataBindCapture bindCapture() throws SQLException {
DataBindCapture dataBind = DataBindCapture.of(dataTimeZone);
if (encryptedProps != null) {
for (STreeProperty encryptedProp : encryptedProps) {
dataBind.setString(encryptedProp.getEncryptKeyAsString());
}
}
return dataBind;
}
int getAsOfTableCount() {
return asOfTableCount;
}
@@ -263,13 +281,15 @@ public class CQueryPlan {
/**
* Register an execution time against this query plan;
*/
void executionTime(long loadedBeanCount, long timeMicros, ObjectGraphNode objectGraphNode) {
boolean executionTime(long loadedBeanCount, long timeMicros, ObjectGraphNode objectGraphNode) {
stats.add(loadedBeanCount, timeMicros, objectGraphNode);
if (objectGraphNode != null) {
// collect stats based on objectGraphNode for lazy loading reporting
server.collectQueryStats(objectGraphNode, loadedBeanCount, timeMicros);
}
return bindCapture.collectFor(timeMicros);
}
/**
@@ -300,4 +320,22 @@ public class CQueryPlan {
public TimedMetric createTimedMetric() {
return MetricFactory.get().createTimedMetric(MetricType.ORM, label);
}
void captureBindForQueryPlan(CQueryPredicates predicates, long executionTimeMicros) {
try {
DataBindCapture capture = bindCapture();
predicates.bind(capture);
bindCapture.setBind(capture.bindCapture(), executionTimeMicros);
} catch (SQLException e) {
logger.error("Error capturing bind values", e);
}
}
public void collectQueryPlan(QueryPlanRequest request) {
if (!getSql().equals(RESULT_SET_BASED_RAW_SQL)) {
bindCapture.collectQueryPlan(request);
}
}
}
@@ -125,8 +125,10 @@ class CQueryRowCount implements SpiProfileTransactionEvent {
rowCount = rset.getInt(1);
executionTimeMicros = (System.nanoTime() - startNano) / 1000L;
queryPlan.executionTime(rowCount, executionTimeMicros, query.getParentNode());
request.slowQueryCheck(executionTimeMicros, rowCount);
if (queryPlan.executionTime(rowCount, executionTimeMicros, query.getParentNode())) {
queryPlan.captureBindForQueryPlan(predicates, executionTimeMicros);
}
getTransaction().profileEvent(this);
return rowCount;
@@ -111,7 +111,9 @@ class CQueryUpdate implements SpiProfileTransactionEvent {
executionTimeMicros = (System.nanoTime() - startNano) / 1000L;
request.slowQueryCheck(executionTimeMicros, rowCount);
queryPlan.executionTime(rowCount, executionTimeMicros, null);
if (queryPlan.executionTime(rowCount, executionTimeMicros, null)) {
queryPlan.captureBindForQueryPlan(predicates, executionTimeMicros);
}
getTransaction().profileEvent(this);
return rowCount;
@@ -0,0 +1,108 @@
package io.ebeaninternal.server.query;
import io.ebean.meta.MetaQueryPlan;
/**
* Captured query plan details.
*/
class DQueryPlanOutput implements MetaQueryPlan {
private final Class<?> beanType;
private final String label;
private final String sql;
private final String bind;
private final String plan;
private String planHash;
private long queryTimeMicros;
private long captureCount;
DQueryPlanOutput(Class<?> beanType, String label, String sql, String bind, String plan) {
this.beanType = beanType;
this.label = label;
this.sql = sql;
this.bind = bind;
this.plan = plan;
}
@Override
public String getQueryPlanHash() {
return planHash;
}
/**
* Return the associated bean.
*/
@Override
public Class<?> getBeanType() {
return beanType;
}
/**
* Return the query label if set.
*/
@Override
public String getLabel() {
return label;
}
/**
* Return the sql of query.
*/
@Override
public String getSql() {
return sql;
}
/**
* Return a description of the bind values used.
*/
@Override
public String getBind() {
return bind;
}
/**
* Return the query plan.
*/
@Override
public String getPlan() {
return plan;
}
/**
* Return the query execution time associated with the capture of bind values used
* to build the query plan.
*/
@Override
public long getQueryTimeMicros() {
return queryTimeMicros;
}
/**
* Return the total count of times bind capture has occurred. We don't want this to be
* massive as that implies a high overhead.
*/
@Override
public long getCaptureCount() {
return captureCount;
}
@Override
public String toString() {
return " BeanType:" + ((beanType == null) ? "" : beanType.getSimpleName()) + " planHash:" + planHash + " label:" + label + " queryTimeMicros:" + queryTimeMicros + " captureCount:" + captureCount + "\n SQL:" + sql + "\nBIND:" + bind + "\nPLAN:" + plan;
}
/**
* Additionally set the query execution time and the number of bind captures.
*/
void with(long queryTimeMicros, long captureCount, String planHash) {
this.queryTimeMicros = queryTimeMicros;
this.captureCount = captureCount;
this.planHash = planHash;
}
}
@@ -0,0 +1,36 @@
package io.ebeaninternal.server.query;
import io.ebean.annotation.Platform;
public final class PlatformQueryPlan {
private static QueryPlanLogger explainLogger = new QueryPlanLoggerExplain();
private static QueryPlanLogger postgresLogger = new QueryPlanLoggerPostgres();
private static QueryPlanLogger sqlServerLogger = new QueryPlanLoggerSqlServer();
private static QueryPlanLogger oracleLogger = new QueryPlanLoggerOracle();
/**
* Returns the logger to log query plans for the given platform.
*/
public static QueryPlanLogger getLogger(Platform platform) {
switch (platform) {
case POSTGRES:
return postgresLogger;
case SQLSERVER:
case SQLSERVER16:
case SQLSERVER17:
return sqlServerLogger;
case ORACLE:
return oracleLogger;
default:
return explainLogger;
}
}
}
@@ -0,0 +1,47 @@
package io.ebeaninternal.server.query;
import io.ebeaninternal.server.type.bindcapture.BindCapture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
public abstract class QueryPlanLogger {
static final Logger queryPlanLog = LoggerFactory.getLogger(QueryPlanLogger.class);
public abstract DQueryPlanOutput logQueryPlan(Connection conn, CQueryPlan plan, BindCapture bind);
DQueryPlanOutput readQueryPlan(CQueryPlan plan, BindCapture bind, ResultSet rset) throws SQLException {
StringBuilder sb = new StringBuilder();
for (int i = 1; i <= rset.getMetaData().getColumnCount(); i++) {
sb.append(rset.getMetaData().getColumnLabel(i)).append("\t");
}
sb.setLength(sb.length() - 1);
readPlanData(sb, rset);
return createPlan(plan, bind.toString(), sb.toString());
}
protected DQueryPlanOutput createPlan(CQueryPlan plan, String bind, String planString) {
return new DQueryPlanOutput(plan.getBeanType(), plan.getLabel(), plan.getSql(), bind, planString);
}
DQueryPlanOutput readQueryPlanBasic(CQueryPlan plan, BindCapture bind, ResultSet rset) throws SQLException {
StringBuilder sb = new StringBuilder();
readPlanData(sb, rset);
return createPlan(plan, bind.toString(), sb.toString().trim());
}
private void readPlanData(StringBuilder sb, ResultSet rset) throws SQLException {
while (rset.next()) {
sb.append('\n');
for (int i = 1; i <= rset.getMetaData().getColumnCount(); i++) {
sb.append(rset.getString(i)).append("\t");
}
sb.setLength(sb.length()-1);
}
}
}
@@ -0,0 +1,30 @@
package io.ebeaninternal.server.query;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import io.ebeaninternal.server.type.bindcapture.BindCapture;
/**
* A QueryPlanlogger that prefixes "EXPLAIN " to the query. This works for Postgres, H2 and MySql.
*/
public class QueryPlanLoggerExplain extends QueryPlanLogger {
@Override
public DQueryPlanOutput logQueryPlan(Connection conn, CQueryPlan plan, BindCapture bind) {
try (PreparedStatement explainStmt = conn.prepareStatement("EXPLAIN " + plan.getSql())) {
bind.prepare(explainStmt, conn);
try (ResultSet rset = explainStmt.executeQuery()) {
return readQueryPlan(plan, bind, rset);
}
} catch (SQLException e) {
queryPlanLog.error("Could not log query plan", e);
}
return null;
}
}
@@ -0,0 +1,37 @@
package io.ebeaninternal.server.query;
import io.ebeaninternal.server.type.bindcapture.BindCapture;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
/**
* A QueryPlanlogger for oracle.
*
* @author Roland Praml, FOCONIS AG
*/
public class QueryPlanLoggerOracle extends QueryPlanLogger {
@Override
public DQueryPlanOutput logQueryPlan(Connection conn, CQueryPlan plan, BindCapture bind) {
try (Statement stmt = conn.createStatement()) {
try (PreparedStatement explainStmt = conn.prepareStatement("EXPLAIN PLAN FOR " + plan.getSql())) {
bind.prepare(explainStmt, conn);
explainStmt.execute();
}
try (ResultSet rset = stmt.executeQuery("select plan_table_output from table(dbms_xplan.display())")) {
return readQueryPlan(plan, bind, rset);
}
} catch (SQLException e) {
queryPlanLog.error("Could not log query plan", e);
}
return null;
}
}
@@ -0,0 +1,31 @@
package io.ebeaninternal.server.query;
import io.ebeaninternal.server.type.bindcapture.BindCapture;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* A QueryPlanlogger that prefixes "EXPLAIN " to the query. This works for Postgres, H2 and MySql.
*/
public class QueryPlanLoggerPostgres extends QueryPlanLogger {
@Override
public DQueryPlanOutput logQueryPlan(Connection conn, CQueryPlan plan, BindCapture bind) {
String explain = "EXPLAIN " + plan.getSql();
try (PreparedStatement explainStmt = conn.prepareStatement(explain)) {
bind.prepare(explainStmt, conn);
try (ResultSet rset = explainStmt.executeQuery()) {
return readQueryPlanBasic(plan, bind, rset);
}
} catch (SQLException e) {
queryPlanLog.error("Could not log query plan: " + explain, e);
throw new IllegalStateException("Failed to obtain explain plan: " + explain, e);
}
}
}
@@ -0,0 +1,52 @@
package io.ebeaninternal.server.query;
import io.ebeaninternal.server.type.bindcapture.BindCapture;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
/**
* A QueryPlanlogger for sqlserver. It will return the plan as XML, which can be opened in
* Microsoft SQL Server Management Studio.
*
* @author Roland Praml, FOCONIS AG
*/
public class QueryPlanLoggerSqlServer extends QueryPlanLogger {
@Override
public DQueryPlanOutput logQueryPlan(Connection conn, CQueryPlan plan, BindCapture bind) {
try (Statement stmt = conn.createStatement()) {
stmt.execute("SET STATISTICS XML ON");
try (PreparedStatement explainStmt = conn.prepareStatement(plan.getSql())) {
bind.prepare(explainStmt, conn);
try (ResultSet rset = explainStmt.executeQuery()) {
// unfortunately, this will execute the
}
if (explainStmt.getMoreResults()) {
try (ResultSet rset = explainStmt.getResultSet()) {
StringBuilder sb = new StringBuilder();
while (rset.next()) {
sb.append("XML: ").append(rset.getString(1));
}
return createPlan(plan, bind.toString(), sb.toString());
}
}
} catch (SQLException e) {
queryPlanLog.error("Could not log query plan", e);
} finally {
stmt.execute("SET STATISTICS XML OFF");
}
} catch (SQLException e) {
queryPlanLog.error("Could not log query plan", e);
}
return null;
}
}