#304 - ElasticSearch integration part 1 / doc store integration

This commit is contained in:
Robin Bygrave committed 2016-03-01 15:36:44 +13:00
1 parent cb1fe297d6
commit 8d3b4a0407
209 files changed
+6736 -573

No files matched your search

@@ -3,6 +3,7 @@ package com.avaje.ebeaninternal.server.transaction;
import com.avaje.ebean.BackgroundExecutor;
import com.avaje.ebean.config.ServerConfig;
import com.avaje.ebeaninternal.api.SpiTransaction;
import com.avaje.ebeanservice.docstore.api.DocStoreUpdateProcessor;
import com.avaje.ebeaninternal.server.cluster.ClusterManager;
import com.avaje.ebeaninternal.server.core.BootupClasses;
import com.avaje.ebeaninternal.server.deploy.BeanDescriptorManager;
@@ -16,10 +17,10 @@ import java.sql.Connection;
*/
public class AutoCommitTransactionManager extends TransactionManager {
public AutoCommitTransactionManager(ClusterManager clusterManager, BackgroundExecutor backgroundExecutor,
ServerConfig config, BeanDescriptorManager descMgr, BootupClasses bootupClasses) {
super(clusterManager, backgroundExecutor, config, descMgr, bootupClasses);
public AutoCommitTransactionManager(ServerConfig serverConfig, ClusterManager clusterManager, BackgroundExecutor backgroundExecutor,
DocStoreUpdateProcessor indexUpdateProcessor, BeanDescriptorManager descMgr, BootupClasses bootupClasses) {
super(serverConfig, clusterManager, backgroundExecutor, indexUpdateProcessor, descMgr, bootupClasses);
}
/**
@@ -23,7 +23,7 @@ import java.util.Set;
* <p>
* Duplicate beans are ones having the same type and unique id value. These are
* considered duplicates and replaced by the bean instance that was already
* loaded into the PersistanceContext.
* loaded into the PersistenceContext.
* </p>
*/
public final class DefaultPersistenceContext implements PersistenceContext {
@@ -36,13 +36,13 @@ public final class DefaultPersistenceContext implements PersistenceContext {
private final Monitor monitor = new Monitor();
/**
* Create a new PersistanceContext.
* Create a new PersistenceContext.
*/
public DefaultPersistenceContext() {
}
/**
* Set an object into the PersistanceContext.
* Set an object into the PersistenceContext.
*/
public void put(Object id, Object bean) {
synchronized (monitor) {
@@ -9,6 +9,10 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.avaje.ebean.annotation.DocStoreEvent;
import com.avaje.ebeanservice.docstore.api.support.DocStoreDeleteEvent;
import com.avaje.ebeanservice.docstore.api.DocStoreUpdates;
/**
* Beans deleted by Id used for updating L2 Cache.
*/
@@ -72,5 +76,28 @@ public final class DeleteByIdMap {
return r;
}
/**
* Add the deletes to the DocStoreUpdates.
*/
public void addDocStoreUpdates(DocStoreUpdates docStoreUpdates, DocStoreEvent txnIndexMode) {
for (BeanPersistIds deleteIds : beanMap.values()) {
BeanDescriptor<?> desc = deleteIds.getBeanDescriptor();
DocStoreEvent docStoreEvent = desc.getDocStoreEvent(PersistRequest.Type.DELETE, txnIndexMode);
if (DocStoreEvent.IGNORE != docStoreEvent) {
// Add to queue or bulk update entries
boolean queue = (DocStoreEvent.QUEUE == docStoreEvent);
String queueId = desc.getDocStoreQueueId();
List<Serializable> idValues = deleteIds.getDeleteIds();
if (idValues != null) {
for (int i = 0; i < idValues.size(); i++) {
if (queue) {
docStoreUpdates.queueDelete(queueId, idValues.get(i));
} else {
docStoreUpdates.addDelete(new DocStoreDeleteEvent(desc, idValues.get(i)));
}
}
}
}
}
}
}
@@ -4,6 +4,7 @@ import com.avaje.ebean.BackgroundExecutor;
import com.avaje.ebean.config.ServerConfig;
import com.avaje.ebean.config.dbplatform.DatabasePlatform;
import com.avaje.ebeaninternal.api.SpiTransaction;
import com.avaje.ebeanservice.docstore.api.DocStoreUpdateProcessor;
import com.avaje.ebeaninternal.server.cluster.ClusterManager;
import com.avaje.ebeaninternal.server.core.BootupClasses;
import com.avaje.ebeaninternal.server.deploy.BeanDescriptorManager;
@@ -16,10 +17,10 @@ import java.sql.Connection;
*/
public class ExplicitTransactionManager extends TransactionManager {
public ExplicitTransactionManager(ClusterManager clusterManager, BackgroundExecutor backgroundExecutor,
ServerConfig config, BeanDescriptorManager descMgr, BootupClasses bootupClasses) {
public ExplicitTransactionManager(ServerConfig serverConfig, ClusterManager clusterManager, BackgroundExecutor backgroundExecutor,
DocStoreUpdateProcessor indexUpdateProcessor, BeanDescriptorManager descMgr, BootupClasses bootupClasses) {
super(clusterManager, backgroundExecutor, config, descMgr, bootupClasses);
super(serverConfig, clusterManager, backgroundExecutor, indexUpdateProcessor, descMgr, bootupClasses);
}
/**
@@ -1,6 +1,7 @@
package com.avaje.ebeaninternal.server.transaction;
import com.avaje.ebean.TransactionCallback;
import com.avaje.ebean.annotation.DocStoreEvent;
import com.avaje.ebean.bean.PersistenceContext;
import com.avaje.ebean.config.PersistBatch;
import com.avaje.ebean.config.dbplatform.DatabasePlatform.OnQueryOnly;
@@ -142,6 +143,14 @@ public class JdbcTransaction implements SpiTransaction {
protected TChangeLogHolder changeLogHolder;
/**
* The mode for updating doc store indexes for this transaction.
* Only set when you want to override the default behavior.
*/
protected DocStoreEvent docStoreUpdateMode;
protected int docStoreBulkBatchSize;
/**
* Create a new JdbcTransaction.
*/
@@ -272,6 +281,25 @@ public class JdbcTransaction implements SpiTransaction {
}
}
@Override
public int getDocStoreBulkBatchSize() {
return docStoreBulkBatchSize;
}
@Override
public void setDocStoreUpdateBatchSize(int docStoreBulkBatchSize) {
this.docStoreBulkBatchSize = docStoreBulkBatchSize;
}
public DocStoreEvent getDocStoreUpdateMode() {
return docStoreUpdateMode;
}
@Override
public void setDocStoreUpdateMode(DocStoreEvent docStoreUpdateMode) {
this.docStoreUpdateMode = docStoreUpdateMode;
}
@Override
public List<DerivedRelationshipData> getDerivedRelationship(Object bean) {
if (derivedRelMap == null) {
@@ -1,8 +1,11 @@
package com.avaje.ebeaninternal.server.transaction;
import com.avaje.ebean.annotation.DocStoreEvent;
import com.avaje.ebeaninternal.api.SpiTransaction;
import com.avaje.ebeaninternal.api.TransactionEvent;
import com.avaje.ebeaninternal.api.TransactionEventTable;
import com.avaje.ebeaninternal.api.TransactionEventTable.TableIUD;
import com.avaje.ebeanservice.docstore.api.DocStoreUpdates;
import com.avaje.ebeaninternal.server.cluster.ClusterManager;
import com.avaje.ebeaninternal.server.core.PersistRequestBean;
import com.avaje.ebeaninternal.server.deploy.BeanDescriptorManager;
@@ -37,14 +40,20 @@ public final class PostCommitProcessing {
private final DeleteByIdMap deleteByIdMap;
private final DocStoreEvent txnDocStoreMode;
private final int txnDocStoreBatchSize;
/**
* Create for a TransactionManager and event.
* Create for an external modification.
*/
public PostCommitProcessing(ClusterManager clusterManager, TransactionManager manager, TransactionEvent event) {
this.clusterManager = clusterManager;
this.manager = manager;
this.serverName = manager.getServerName();
this.txnDocStoreMode = DocStoreEvent.IGNORE;
this.txnDocStoreBatchSize = 0;
this.event = event;
this.deleteByIdMap = event.getDeleteByIdMap();
this.persistBeanRequests = event.getPersistRequestBeans();
@@ -52,6 +61,23 @@ public final class PostCommitProcessing {
this.remoteTransactionEvent = createRemoteTransactionEvent();
}
/**
* Create for a transaction.
*/
public PostCommitProcessing(ClusterManager clusterManager, TransactionManager manager, SpiTransaction transaction) {
this.clusterManager = clusterManager;
this.manager = manager;
this.serverName = manager.getServerName();
this.txnDocStoreMode = transaction.getDocStoreUpdateMode();
this.txnDocStoreBatchSize = transaction.getDocStoreBulkBatchSize();
this.event = transaction.getEvent();
this.deleteByIdMap = event.getDeleteByIdMap();
this.persistBeanRequests = event.getPersistRequestBeans();
this.beanPersistIdMap = createBeanPersistIdMap();
this.remoteTransactionEvent = createRemoteTransactionEvent();
}
public void notifyLocalCacheIndex() {
// notify cache with bulk insert/update/delete statements
@@ -76,6 +102,33 @@ public final class PostCommitProcessing {
}
}
/**
* Process any document store updates.
*/
protected void processDocStoreUpdates() {
if (isDocStoreUpdate()) {
// collect 'bulk update' and 'queue' events
DocStoreUpdates docStoreUpdates = new DocStoreUpdates();
event.addDocStoreUpdates(docStoreUpdates);
if (deleteByIdMap != null) {
deleteByIdMap.addDocStoreUpdates(docStoreUpdates, txnDocStoreMode);
}
if (!docStoreUpdates.isEmpty()) {
// send to docstore / ElasticSearch and/or queue
manager.processDocStoreUpdates(docStoreUpdates, txnDocStoreBatchSize);
}
}
}
/**
* Return true if updates to the document store occur for this transaction.
*/
private boolean isDocStoreUpdate() {
return manager.isDocStoreActive() && (txnDocStoreMode == null || txnDocStoreMode != DocStoreEvent.IGNORE);
}
public void notifyCluster() {
if (remoteTransactionEvent != null && !remoteTransactionEvent.isEmpty()) {
// send the interesting events to the cluster
@@ -91,6 +144,7 @@ public final class PostCommitProcessing {
return new Runnable() {
public void run() {
localPersistListenersNotify();
processDocStoreUpdates();
}
};
}
@@ -17,6 +17,8 @@ import com.avaje.ebeaninternal.server.cluster.ClusterManager;
import com.avaje.ebeaninternal.server.core.BootupClasses;
import com.avaje.ebeaninternal.server.deploy.BeanDescriptorManager;
import com.avaje.ebeaninternal.server.lib.sql.DataSourcePool;
import com.avaje.ebeanservice.docstore.api.DocStoreUpdateProcessor;
import com.avaje.ebeanservice.docstore.api.DocStoreUpdates;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,6 +72,13 @@ public class TransactionManager {
protected final String serverName;
protected final boolean docStoreActive;
/**
* The elastic search index update processor.
*/
protected final DocStoreUpdateProcessor docStoreUpdateProcessor;
protected final PersistBatch persistBatch;
protected final PersistBatch persistBatchOnCascade;
@@ -97,7 +106,7 @@ public class TransactionManager {
/**
* Create the TransactionManager
*/
public TransactionManager(ClusterManager clusterManager, BackgroundExecutor backgroundExecutor, ServerConfig config,
public TransactionManager(ServerConfig config, ClusterManager clusterManager, BackgroundExecutor backgroundExecutor, DocStoreUpdateProcessor docStoreUpdateProcessor,
BeanDescriptorManager descMgr, BootupClasses bootupClasses) {
this.persistBatch = config.getPersistBatch();
@@ -109,11 +118,12 @@ public class TransactionManager {
this.serverName = config.getName();
this.backgroundExecutor = backgroundExecutor;
this.dataSource = config.getDataSource();
this.docStoreActive = config.getDocStoreConfig().isActive();
this.docStoreUpdateProcessor = docStoreUpdateProcessor;
this.bulkEventListenerMap = new BulkEventListenerMap(config.getBulkTableEventListeners());
List<TransactionEventListener> transactionEventListeners = bootupClasses.getTransactionEventListeners();
this.transactionEventListeners = transactionEventListeners.toArray(new
TransactionEventListener[transactionEventListeners.size()]);
this.transactionEventListeners = transactionEventListeners.toArray(new TransactionEventListener[transactionEventListeners.size()]);
this.prefix = "";
this.externalTransPrefix = "e";
@@ -136,6 +146,10 @@ public class TransactionManager {
}
}
public boolean isDocStoreActive() {
return docStoreActive;
}
public BeanDescriptorManager getBeanDescriptorManager() {
return beanDescriptorManager;
}
@@ -391,7 +405,7 @@ public class TransactionManager {
TXN_LOGGER.debug(transaction.getLogPrefix() + "Commit");
}
PostCommitProcessing postCommit = new PostCommitProcessing(clusterManager, this, transaction.getEvent());
PostCommitProcessing postCommit = new PostCommitProcessing(clusterManager, this, transaction);
postCommit.notifyLocalCacheIndex();
postCommit.notifyCluster();
@@ -454,6 +468,13 @@ public class TransactionManager {
}
}
/**
* Process the docstore / ElasticSearch updates.
*/
public void processDocStoreUpdates(DocStoreUpdates docStoreUpdates, int bulkBatchSize) {
docStoreUpdateProcessor.process(docStoreUpdates, bulkBatchSize);
}
/**
* Prepare and then send/log the changeSet.
*/