mirror of
https://github.com/ebean-orm/ebean.git
synced 2026-11-18 14:58:47 +00:00
#304 - ElasticSearch integration part 1 / doc store integration
This commit is contained in:
1 parent
cb1fe297d6
commit
8d3b4a0407
209 files changed
+6736
-573
No files matched your search
+5
-4
@@ -3,6 +3,7 @@ package com.avaje.ebeaninternal.server.transaction;
|
||||
import com.avaje.ebean.BackgroundExecutor;
|
||||
import com.avaje.ebean.config.ServerConfig;
|
||||
import com.avaje.ebeaninternal.api.SpiTransaction;
|
||||
import com.avaje.ebeanservice.docstore.api.DocStoreUpdateProcessor;
|
||||
import com.avaje.ebeaninternal.server.cluster.ClusterManager;
|
||||
import com.avaje.ebeaninternal.server.core.BootupClasses;
|
||||
import com.avaje.ebeaninternal.server.deploy.BeanDescriptorManager;
|
||||
@@ -16,10 +17,10 @@ import java.sql.Connection;
|
||||
*/
|
||||
public class AutoCommitTransactionManager extends TransactionManager {
|
||||
|
||||
public AutoCommitTransactionManager(ClusterManager clusterManager, BackgroundExecutor backgroundExecutor,
|
||||
ServerConfig config, BeanDescriptorManager descMgr, BootupClasses bootupClasses) {
|
||||
|
||||
super(clusterManager, backgroundExecutor, config, descMgr, bootupClasses);
|
||||
public AutoCommitTransactionManager(ServerConfig serverConfig, ClusterManager clusterManager, BackgroundExecutor backgroundExecutor,
|
||||
DocStoreUpdateProcessor indexUpdateProcessor, BeanDescriptorManager descMgr, BootupClasses bootupClasses) {
|
||||
|
||||
super(serverConfig, clusterManager, backgroundExecutor, indexUpdateProcessor, descMgr, bootupClasses);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+3
-3
@@ -23,7 +23,7 @@ import java.util.Set;
|
||||
* <p>
|
||||
* Duplicate beans are ones having the same type and unique id value. These are
|
||||
* considered duplicates and replaced by the bean instance that was already
|
||||
* loaded into the PersistanceContext.
|
||||
* loaded into the PersistenceContext.
|
||||
* </p>
|
||||
*/
|
||||
public final class DefaultPersistenceContext implements PersistenceContext {
|
||||
@@ -36,13 +36,13 @@ public final class DefaultPersistenceContext implements PersistenceContext {
|
||||
private final Monitor monitor = new Monitor();
|
||||
|
||||
/**
|
||||
* Create a new PersistanceContext.
|
||||
* Create a new PersistenceContext.
|
||||
*/
|
||||
public DefaultPersistenceContext() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Set an object into the PersistanceContext.
|
||||
* Set an object into the PersistenceContext.
|
||||
*/
|
||||
public void put(Object id, Object bean) {
|
||||
synchronized (monitor) {
|
||||
|
||||
@@ -9,6 +9,10 @@ import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.avaje.ebean.annotation.DocStoreEvent;
|
||||
import com.avaje.ebeanservice.docstore.api.support.DocStoreDeleteEvent;
|
||||
import com.avaje.ebeanservice.docstore.api.DocStoreUpdates;
|
||||
|
||||
/**
|
||||
* Beans deleted by Id used for updating L2 Cache.
|
||||
*/
|
||||
@@ -72,5 +76,28 @@ public final class DeleteByIdMap {
|
||||
return r;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Add the deletes to the DocStoreUpdates.
|
||||
*/
|
||||
public void addDocStoreUpdates(DocStoreUpdates docStoreUpdates, DocStoreEvent txnIndexMode) {
|
||||
for (BeanPersistIds deleteIds : beanMap.values()) {
|
||||
BeanDescriptor<?> desc = deleteIds.getBeanDescriptor();
|
||||
DocStoreEvent docStoreEvent = desc.getDocStoreEvent(PersistRequest.Type.DELETE, txnIndexMode);
|
||||
if (DocStoreEvent.IGNORE != docStoreEvent) {
|
||||
// Add to queue or bulk update entries
|
||||
boolean queue = (DocStoreEvent.QUEUE == docStoreEvent);
|
||||
String queueId = desc.getDocStoreQueueId();
|
||||
List<Serializable> idValues = deleteIds.getDeleteIds();
|
||||
if (idValues != null) {
|
||||
for (int i = 0; i < idValues.size(); i++) {
|
||||
if (queue) {
|
||||
docStoreUpdates.queueDelete(queueId, idValues.get(i));
|
||||
} else {
|
||||
docStoreUpdates.addDelete(new DocStoreDeleteEvent(desc, idValues.get(i)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
-3
@@ -4,6 +4,7 @@ import com.avaje.ebean.BackgroundExecutor;
|
||||
import com.avaje.ebean.config.ServerConfig;
|
||||
import com.avaje.ebean.config.dbplatform.DatabasePlatform;
|
||||
import com.avaje.ebeaninternal.api.SpiTransaction;
|
||||
import com.avaje.ebeanservice.docstore.api.DocStoreUpdateProcessor;
|
||||
import com.avaje.ebeaninternal.server.cluster.ClusterManager;
|
||||
import com.avaje.ebeaninternal.server.core.BootupClasses;
|
||||
import com.avaje.ebeaninternal.server.deploy.BeanDescriptorManager;
|
||||
@@ -16,10 +17,10 @@ import java.sql.Connection;
|
||||
*/
|
||||
public class ExplicitTransactionManager extends TransactionManager {
|
||||
|
||||
public ExplicitTransactionManager(ClusterManager clusterManager, BackgroundExecutor backgroundExecutor,
|
||||
ServerConfig config, BeanDescriptorManager descMgr, BootupClasses bootupClasses) {
|
||||
public ExplicitTransactionManager(ServerConfig serverConfig, ClusterManager clusterManager, BackgroundExecutor backgroundExecutor,
|
||||
DocStoreUpdateProcessor indexUpdateProcessor, BeanDescriptorManager descMgr, BootupClasses bootupClasses) {
|
||||
|
||||
super(clusterManager, backgroundExecutor, config, descMgr, bootupClasses);
|
||||
super(serverConfig, clusterManager, backgroundExecutor, indexUpdateProcessor, descMgr, bootupClasses);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.avaje.ebeaninternal.server.transaction;
|
||||
|
||||
import com.avaje.ebean.TransactionCallback;
|
||||
import com.avaje.ebean.annotation.DocStoreEvent;
|
||||
import com.avaje.ebean.bean.PersistenceContext;
|
||||
import com.avaje.ebean.config.PersistBatch;
|
||||
import com.avaje.ebean.config.dbplatform.DatabasePlatform.OnQueryOnly;
|
||||
@@ -142,6 +143,14 @@ public class JdbcTransaction implements SpiTransaction {
|
||||
|
||||
protected TChangeLogHolder changeLogHolder;
|
||||
|
||||
/**
|
||||
* The mode for updating doc store indexes for this transaction.
|
||||
* Only set when you want to override the default behavior.
|
||||
*/
|
||||
protected DocStoreEvent docStoreUpdateMode;
|
||||
|
||||
protected int docStoreBulkBatchSize;
|
||||
|
||||
/**
|
||||
* Create a new JdbcTransaction.
|
||||
*/
|
||||
@@ -272,6 +281,25 @@ public class JdbcTransaction implements SpiTransaction {
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getDocStoreBulkBatchSize() {
|
||||
return docStoreBulkBatchSize;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setDocStoreUpdateBatchSize(int docStoreBulkBatchSize) {
|
||||
this.docStoreBulkBatchSize = docStoreBulkBatchSize;
|
||||
}
|
||||
|
||||
public DocStoreEvent getDocStoreUpdateMode() {
|
||||
return docStoreUpdateMode;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setDocStoreUpdateMode(DocStoreEvent docStoreUpdateMode) {
|
||||
this.docStoreUpdateMode = docStoreUpdateMode;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DerivedRelationshipData> getDerivedRelationship(Object bean) {
|
||||
if (derivedRelMap == null) {
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
package com.avaje.ebeaninternal.server.transaction;
|
||||
|
||||
import com.avaje.ebean.annotation.DocStoreEvent;
|
||||
import com.avaje.ebeaninternal.api.SpiTransaction;
|
||||
import com.avaje.ebeaninternal.api.TransactionEvent;
|
||||
import com.avaje.ebeaninternal.api.TransactionEventTable;
|
||||
import com.avaje.ebeaninternal.api.TransactionEventTable.TableIUD;
|
||||
import com.avaje.ebeanservice.docstore.api.DocStoreUpdates;
|
||||
import com.avaje.ebeaninternal.server.cluster.ClusterManager;
|
||||
import com.avaje.ebeaninternal.server.core.PersistRequestBean;
|
||||
import com.avaje.ebeaninternal.server.deploy.BeanDescriptorManager;
|
||||
@@ -37,14 +40,20 @@ public final class PostCommitProcessing {
|
||||
|
||||
private final DeleteByIdMap deleteByIdMap;
|
||||
|
||||
private final DocStoreEvent txnDocStoreMode;
|
||||
|
||||
private final int txnDocStoreBatchSize;
|
||||
|
||||
/**
|
||||
* Create for a TransactionManager and event.
|
||||
* Create for an external modification.
|
||||
*/
|
||||
public PostCommitProcessing(ClusterManager clusterManager, TransactionManager manager, TransactionEvent event) {
|
||||
|
||||
this.clusterManager = clusterManager;
|
||||
this.manager = manager;
|
||||
this.serverName = manager.getServerName();
|
||||
this.txnDocStoreMode = DocStoreEvent.IGNORE;
|
||||
this.txnDocStoreBatchSize = 0;
|
||||
this.event = event;
|
||||
this.deleteByIdMap = event.getDeleteByIdMap();
|
||||
this.persistBeanRequests = event.getPersistRequestBeans();
|
||||
@@ -52,6 +61,23 @@ public final class PostCommitProcessing {
|
||||
this.remoteTransactionEvent = createRemoteTransactionEvent();
|
||||
}
|
||||
|
||||
/**
|
||||
* Create for a transaction.
|
||||
*/
|
||||
public PostCommitProcessing(ClusterManager clusterManager, TransactionManager manager, SpiTransaction transaction) {
|
||||
|
||||
this.clusterManager = clusterManager;
|
||||
this.manager = manager;
|
||||
this.serverName = manager.getServerName();
|
||||
this.txnDocStoreMode = transaction.getDocStoreUpdateMode();
|
||||
this.txnDocStoreBatchSize = transaction.getDocStoreBulkBatchSize();
|
||||
this.event = transaction.getEvent();
|
||||
this.deleteByIdMap = event.getDeleteByIdMap();
|
||||
this.persistBeanRequests = event.getPersistRequestBeans();
|
||||
this.beanPersistIdMap = createBeanPersistIdMap();
|
||||
this.remoteTransactionEvent = createRemoteTransactionEvent();
|
||||
}
|
||||
|
||||
public void notifyLocalCacheIndex() {
|
||||
|
||||
// notify cache with bulk insert/update/delete statements
|
||||
@@ -76,6 +102,33 @@ public final class PostCommitProcessing {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process any document store updates.
|
||||
*/
|
||||
protected void processDocStoreUpdates() {
|
||||
|
||||
if (isDocStoreUpdate()) {
|
||||
// collect 'bulk update' and 'queue' events
|
||||
DocStoreUpdates docStoreUpdates = new DocStoreUpdates();
|
||||
event.addDocStoreUpdates(docStoreUpdates);
|
||||
if (deleteByIdMap != null) {
|
||||
deleteByIdMap.addDocStoreUpdates(docStoreUpdates, txnDocStoreMode);
|
||||
}
|
||||
|
||||
if (!docStoreUpdates.isEmpty()) {
|
||||
// send to docstore / ElasticSearch and/or queue
|
||||
manager.processDocStoreUpdates(docStoreUpdates, txnDocStoreBatchSize);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return true if updates to the document store occur for this transaction.
|
||||
*/
|
||||
private boolean isDocStoreUpdate() {
|
||||
return manager.isDocStoreActive() && (txnDocStoreMode == null || txnDocStoreMode != DocStoreEvent.IGNORE);
|
||||
}
|
||||
|
||||
public void notifyCluster() {
|
||||
if (remoteTransactionEvent != null && !remoteTransactionEvent.isEmpty()) {
|
||||
// send the interesting events to the cluster
|
||||
@@ -91,6 +144,7 @@ public final class PostCommitProcessing {
|
||||
return new Runnable() {
|
||||
public void run() {
|
||||
localPersistListenersNotify();
|
||||
processDocStoreUpdates();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -17,6 +17,8 @@ import com.avaje.ebeaninternal.server.cluster.ClusterManager;
|
||||
import com.avaje.ebeaninternal.server.core.BootupClasses;
|
||||
import com.avaje.ebeaninternal.server.deploy.BeanDescriptorManager;
|
||||
import com.avaje.ebeaninternal.server.lib.sql.DataSourcePool;
|
||||
import com.avaje.ebeanservice.docstore.api.DocStoreUpdateProcessor;
|
||||
import com.avaje.ebeanservice.docstore.api.DocStoreUpdates;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
@@ -70,6 +72,13 @@ public class TransactionManager {
|
||||
|
||||
protected final String serverName;
|
||||
|
||||
protected final boolean docStoreActive;
|
||||
|
||||
/**
|
||||
* The elastic search index update processor.
|
||||
*/
|
||||
protected final DocStoreUpdateProcessor docStoreUpdateProcessor;
|
||||
|
||||
protected final PersistBatch persistBatch;
|
||||
|
||||
protected final PersistBatch persistBatchOnCascade;
|
||||
@@ -97,7 +106,7 @@ public class TransactionManager {
|
||||
/**
|
||||
* Create the TransactionManager
|
||||
*/
|
||||
public TransactionManager(ClusterManager clusterManager, BackgroundExecutor backgroundExecutor, ServerConfig config,
|
||||
public TransactionManager(ServerConfig config, ClusterManager clusterManager, BackgroundExecutor backgroundExecutor, DocStoreUpdateProcessor docStoreUpdateProcessor,
|
||||
BeanDescriptorManager descMgr, BootupClasses bootupClasses) {
|
||||
|
||||
this.persistBatch = config.getPersistBatch();
|
||||
@@ -109,11 +118,12 @@ public class TransactionManager {
|
||||
this.serverName = config.getName();
|
||||
this.backgroundExecutor = backgroundExecutor;
|
||||
this.dataSource = config.getDataSource();
|
||||
this.docStoreActive = config.getDocStoreConfig().isActive();
|
||||
this.docStoreUpdateProcessor = docStoreUpdateProcessor;
|
||||
this.bulkEventListenerMap = new BulkEventListenerMap(config.getBulkTableEventListeners());
|
||||
|
||||
List<TransactionEventListener> transactionEventListeners = bootupClasses.getTransactionEventListeners();
|
||||
this.transactionEventListeners = transactionEventListeners.toArray(new
|
||||
TransactionEventListener[transactionEventListeners.size()]);
|
||||
this.transactionEventListeners = transactionEventListeners.toArray(new TransactionEventListener[transactionEventListeners.size()]);
|
||||
|
||||
this.prefix = "";
|
||||
this.externalTransPrefix = "e";
|
||||
@@ -136,6 +146,10 @@ public class TransactionManager {
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isDocStoreActive() {
|
||||
return docStoreActive;
|
||||
}
|
||||
|
||||
public BeanDescriptorManager getBeanDescriptorManager() {
|
||||
return beanDescriptorManager;
|
||||
}
|
||||
@@ -391,7 +405,7 @@ public class TransactionManager {
|
||||
TXN_LOGGER.debug(transaction.getLogPrefix() + "Commit");
|
||||
}
|
||||
|
||||
PostCommitProcessing postCommit = new PostCommitProcessing(clusterManager, this, transaction.getEvent());
|
||||
PostCommitProcessing postCommit = new PostCommitProcessing(clusterManager, this, transaction);
|
||||
|
||||
postCommit.notifyLocalCacheIndex();
|
||||
postCommit.notifyCluster();
|
||||
@@ -454,6 +468,13 @@ public class TransactionManager {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the docstore / ElasticSearch updates.
|
||||
*/
|
||||
public void processDocStoreUpdates(DocStoreUpdates docStoreUpdates, int bulkBatchSize) {
|
||||
docStoreUpdateProcessor.process(docStoreUpdates, bulkBatchSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare and then send/log the changeSet.
|
||||
*/
|
||||
|
||||
Reference in new issue
Block a user