#304 - ElasticSearch integration part 1 / doc store integration

This commit is contained in:
Robin Bygrave committed 2016-03-01 15:36:44 +13:00
1 parent cb1fe297d6
commit 8d3b4a0407
209 files changed
+6736 -573

No files matched your search

@@ -0,0 +1,116 @@
package com.avaje.ebeanservice.docstore.api;
import com.avaje.ebean.Query;
import com.avaje.ebean.annotation.DocStoreEvent;
import com.avaje.ebean.plugin.BeanDocType;
import com.avaje.ebeaninternal.server.core.PersistRequest;
import com.avaje.ebeaninternal.server.core.PersistRequestBean;
import com.avaje.ebeanservice.docstore.api.mapping.DocumentMapping;
import java.io.IOException;
import java.util.Set;
/**
* Doc store specific adapter to process doc store events for a given bean type.
*/
public interface DocStoreBeanAdapter<T> extends BeanDocType<T> {
/**
* In deployment phase read the embedded/nested document information.
*/
void registerPaths();
/**
* Register invalidation events for embedded/nested documents the given path and properties.
*/
void registerInvalidationPath(String queueId, String path, Set<String> properties);
/**
* Apply the document structure to the query so that it fetches the required properties to build
* the document (typically in JSON form).
*/
void applyPath(Query<T> query);
/**
* Return true if this type is mapped for doc storage.
*/
boolean isMapped();
/**
* Return the unique queueId for this bean type. This is expected to be a relatively short unique
* string (rather than a fully qualified class name).
*/
String getQueueId();
/**
* Determine and return how this persist type will be processed given the transaction mode.
* <p>
* Some transactions (like bulk updates) might specifically turn off indexing for example.
*/
DocStoreEvent getEvent(PersistRequest.Type persistType, DocStoreEvent txnMode);
/**
* Return the index type for this bean type.
*/
String getIndexType();
/**
* Return the index name for this bean type.
*/
String getIndexName();
/**
* Process a delete by id of a given document.
*/
void deleteById(Object idValue, DocStoreUpdateContext txn) throws IOException;
/**
* Process an index event which is effectively an insert or update (or put).
*/
void index(Object idValue, T entityBean, DocStoreUpdateContext txn) throws IOException;
/**
* Process an insert persist request.
*/
void insert(Object idValue, PersistRequestBean<T> persistRequest, DocStoreUpdateContext txn) throws IOException;
/**
* Process an update persist request.
*/
void update(Object idValue, PersistRequestBean<T> persistRequest, DocStoreUpdateContext txn) throws IOException;
/**
* Process the persist request adding any embedded/nested document invalidation to the docStoreUpdates.
* <p>
* This is expected to check the specific properties to see what other documents they are nested in
* and register invalidation events based on that.
*
* @param request The persist request
* @param docStoreUpdates Invalidation events are registered to this docStoreUpdates
*/
void updateEmbedded(PersistRequestBean<T> request, DocStoreUpdates docStoreUpdates);
/**
* Process an update of an embedded document.
*
* @param idValue the id of the bean effected by an embedded document update
* @param embeddedProperty the path of the property
* @param embeddedRawContent the embedded content for this property in JSON form
* @param txn the doc store transaction to use to process the update
*/
void updateEmbedded(Object idValue, String embeddedProperty, String embeddedRawContent, DocStoreUpdateContext txn) throws IOException;
/**
* Create the document mapping.
*/
DocumentMapping createDocMapping();
/**
* Return an un-analysed property to use instead of the given property.
* <p>
* For analysed properties that we want to sort on we will map the property to an additional
* 'raw' property that we can use for sorting etc.
* </p>
*/
String rawProperty(String property);
}
@@ -0,0 +1,22 @@
package com.avaje.ebeanservice.docstore.api;
import com.avaje.ebean.plugin.SpiServer;
import com.avaje.ebeaninternal.server.deploy.BeanDescriptor;
import com.avaje.ebeaninternal.server.deploy.meta.DeployBeanDescriptor;
/**
* Creates the integration components for DocStore integration.
*/
public interface DocStoreFactory {
/**
* Create and return the DocStore integration components.
*/
DocStoreIntegration create(SpiServer server);
/**
* Create the doc store specific adapter for the given bean type.
*/
<T> DocStoreBeanAdapter<T> createAdapter(BeanDescriptor<T> desc, DeployBeanDescriptor<T> deploy);
}
@@ -0,0 +1,20 @@
package com.avaje.ebeanservice.docstore.api;
import com.avaje.ebean.DocumentStore;
/**
* All the required features for DocStore integration.
*/
public interface DocStoreIntegration {
/**
* Return the DocStoreUpdateProcessor to use.
*/
DocStoreUpdateProcessor updateProcessor();
/**
* Return the DocStore.
*/
DocumentStore documentStore();
}
@@ -0,0 +1,23 @@
package com.avaje.ebeanservice.docstore.api;
import java.io.IOException;
/**
* Update the document store using a Ebean ORM query.
* <p>
* Executes a forEach query and updates the document store with the bean object graph returned by the query.
* </p>
*/
public interface DocStoreQueryUpdate<T> {
/**
* Process the bean storing in the document store.
*/
void store(Object idValue, T bean) throws IOException;
/**
* Flush the changes to the document store.
*/
void flush() throws IOException;
}
@@ -0,0 +1,19 @@
package com.avaje.ebeanservice.docstore.api;
import java.io.IOException;
/**
* For persist events that know how to publish or queue their change to the Document store.
*/
public interface DocStoreUpdate {
/**
* Add the event to the doc store bulk update.
*/
void docStoreUpdate(DocStoreUpdateContext txn) throws IOException;
/**
* Add to the queue for deferred processing.
*/
void addToQueue(DocStoreUpdates docStoreUpdates);
}
@@ -0,0 +1,11 @@
package com.avaje.ebeanservice.docstore.api;
/**
* The doc store specific context/transaction used to collect updates to send to the document store.
* <p>
* Doc store specific implementations gather changes and bulk update the document store.
* </p>
*/
public interface DocStoreUpdateContext {
}
@@ -0,0 +1,34 @@
package com.avaje.ebeanservice.docstore.api;
import com.avaje.ebean.plugin.BeanType;
import java.io.IOException;
/**
* Processes index updates.
* <p>
* This involves sending updates directly to ElasticSearch via it's Bulk API or
* queuing events for future processing.
* </p>
*/
public interface DocStoreUpdateProcessor {
/**
* Create a processor to handle updates per bean via a findEach query.
*/
<T> DocStoreQueryUpdate<T> createQueryUpdate(BeanType<T> beanType, int bulkBatchSize) throws IOException;
/**
* Process all the updates for a transaction.
* <p>
* Typically this makes calls to the Bulk API of the document store or simply adds entries
* to a queue for future processing.
* </p>
*
* @param docStoreUpdates The 'Bulk' and 'Queue' updates to the indexes for the transaction.
* @param bulkBatchSize The batch size to use for Bulk API calls specified on the transaction.
* If this is 0 then the default batch size is used.
*/
void process(DocStoreUpdates docStoreUpdates, int bulkBatchSize);
}
@@ -0,0 +1,115 @@
package com.avaje.ebeanservice.docstore.api;
import com.avaje.ebean.DocStoreQueueEntry;
import com.avaje.ebean.DocStoreQueueEntry.Action;
import java.util.ArrayList;
import java.util.List;
/**
* Collection of document store updates that are either sent to the document store
* or queued for future processing
*/
public class DocStoreUpdates {
/**
* Persist inserts and updates.
*/
private final List<DocStoreUpdate> persistEvents = new ArrayList<DocStoreUpdate>();
/**
* Delete by Id.
*/
private final List<DocStoreUpdate> deleteEvents = new ArrayList<DocStoreUpdate>();
/**
* Nested updates.
*/
private final List<DocStoreQueueEntry> nestedEvents = new ArrayList<DocStoreQueueEntry>();
/**
* Entries sent to the queue for later processing.
*/
private final List<DocStoreQueueEntry> queueEntries = new ArrayList<DocStoreQueueEntry>();
public DocStoreUpdates() {
}
/**
* Return true if there are no events to process.
*/
public boolean isEmpty() {
return persistEvents.isEmpty() && deleteEvents.isEmpty() && nestedEvents.isEmpty() && queueEntries.isEmpty();
}
/**
* Add a persist request.
*/
public void addPersist(DocStoreUpdate bulkRequest) {
persistEvents.add(bulkRequest);
}
/**
* Add a delete request.
*/
public void addDelete(DocStoreUpdate bulkRequest) {
deleteEvents.add(bulkRequest);
}
/**
* Add a nested update.
*/
public void addNested(String queueId, String path, Object beanId) {
nestedEvents.add(new DocStoreQueueEntry(Action.NESTED, queueId, path, beanId));
}
/**
* Queue an 'index' request.
*/
public void queueIndex(String queueId, Object beanId) {
queueEntries.add(new DocStoreQueueEntry(Action.INDEX, queueId, beanId));
}
/**
* Queue a 'delete' request.
*/
public void queueDelete(String queueId, Object beanId) {
queueEntries.add(new DocStoreQueueEntry(Action.DELETE, queueId, beanId));
}
/**
* Queue an update to a nested/embedded object.
*/
public void queueNested(String queueId, String path, Object beanId) {
queueEntries.add(new DocStoreQueueEntry(Action.NESTED, queueId, path, beanId));
}
/**
* Return the persist insert and update requests to be sent to the document store.
*/
public List<DocStoreUpdate> getPersistEvents() {
return persistEvents;
}
/**
* Return delete events.
*/
public List<DocStoreUpdate> getDeleteEvents() {
return deleteEvents;
}
/**
* Return the list of nested update events.
*/
public List<DocStoreQueueEntry> getNestedEvents() {
return nestedEvents;
}
/**
* Return the entries for sending to the queue.
*/
public List<DocStoreQueueEntry> getQueueEntries() {
return queueEntries;
}
}
@@ -0,0 +1,15 @@
package com.avaje.ebeanservice.docstore.api;
/**
* Can be thrown when a document is unexpectedly not found in a document store.
*/
public class DocumentNotFoundException extends RuntimeException {
/**
* Construct with a message.
*/
public DocumentNotFoundException(String message) {
super(message);
}
}
@@ -0,0 +1,143 @@
package com.avaje.ebeanservice.docstore.api.mapping;
import com.avaje.ebean.annotation.DocMapping;
import com.avaje.ebean.annotation.DocStore;
import com.avaje.ebean.text.PathProperties;
import com.avaje.ebeaninternal.server.query.SplitName;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Stack;
/**
* Builds the DocumentMapping for a given bean type.
*/
public class DocMappingBuilder {
private final PathProperties paths;
private final DocStore docStore;
private final Stack<DocPropertyMapping> properties = new Stack<DocPropertyMapping>();
private final Map<String,DocPropertyMapping> map = new LinkedHashMap<String,DocPropertyMapping>();
/**
* Create with the document structure paths and docStore deployment annotation.
*/
public DocMappingBuilder(PathProperties paths, DocStore docStore) {
this.paths = paths;
this.docStore = docStore;
this.properties.push(new DocPropertyMapping());
}
/**
* Return true if the property is included in the document.
*/
public boolean includesProperty(String prefix, String name) {
return paths.includesProperty(prefix, name);
}
/**
* Return true if the path is included in the document.
*/
public boolean includesPath(String prefix, String name) {
return paths.includesProperty(prefix, name);
}
/**
* Add the property mapping.
*/
public void add(DocPropertyMapping docMapping) {
DocPropertyMapping currentParent = properties.peek();
currentParent.addChild(docMapping);
String parentName = currentParent.getName();
String fullName = SplitName.add(parentName, docMapping.getName());
map.put(fullName, docMapping);
}
/**
* Push the nested object or list onto the properties stack.
*/
public void push(DocPropertyMapping nested) {
properties.push(nested);
}
/**
* Pop the nested object or list off the properties stack.
*/
public void pop() {
properties.pop();
}
/**
* Apply any override mappings from the top level docStore annotation.
*/
public void applyMapping() {
DocMapping[] mapping = docStore.mapping();
for (DocMapping docMapping : mapping) {
applyFieldMapping(null, docMapping);
}
}
private void applyFieldMapping(String prefix, DocMapping docMapping) {
String name = docMapping.name();
String fullName = SplitName.add(prefix, name);
DocPropertyMapping mapping = map.get(fullName);
if (mapping == null) {
throw new IllegalStateException("DocMapping for ["+fullName+"] but property not included in document?");
}
mapping.apply(docMapping);
}
/**
* Collect the mapping of properties to 'raw' properties for those marked as sortable.
*/
public Map<String, String> collectSortable() {
DocPropertyMapping peek = properties.peek();
SortableVisitor visitor = new SortableVisitor();
peek.visit(visitor);
return visitor.getSortableMap();
}
/**
* Create the document mapping.
*/
public DocumentMapping create(String queueId, String indexName, String indexType) {
int shards = docStore.shards();
int replicas = docStore.replicas();
DocPropertyMapping root = properties.peek();
return new DocumentMapping(queueId, indexName, indexType, paths, root, shards, replicas);
}
/**
* Find sortable properties to build the mapping to 'raw' properties.
*/
private static class SortableVisitor extends DocPropertyAdapter {
private Map<String,String> sortableMap = new LinkedHashMap<String, String>();
@Override
public void visitProperty(DocPropertyMapping property) {
DocPropertyOptions options = property.getOptions();
if (options != null && Boolean.TRUE.equals(options.getSortable())) {
String fullPath = pathStack.peekFullPath(property.getName());
sortableMap.put(fullPath, fullPath+".raw");
}
}
private Map<String, String> getSortableMap() {
return sortableMap;
}
}
}
@@ -0,0 +1,47 @@
package com.avaje.ebeanservice.docstore.api.mapping;
import com.avaje.ebeaninternal.server.text.json.PathStack;
/**
* Adapter for DocPropertyVisitor that does not do anything.
* Used to extend and implement only the desired methods.
*/
public abstract class DocPropertyAdapter implements DocPropertyVisitor {
protected PathStack pathStack = new PathStack();
@Override
public void visitProperty(DocPropertyMapping property) {
// do nothing
}
@Override
public void visitBegin() {
// do nothing
}
@Override
public void visitEnd() {
// do nothing
}
@Override
public void visitBeginObject(DocPropertyMapping property) {
pathStack.push(property.getName());
}
@Override
public void visitEndObject(DocPropertyMapping property) {
pathStack.pop();
}
@Override
public void visitBeginList(DocPropertyMapping property) {
pathStack.push(property.getName());
}
@Override
public void visitEndList(DocPropertyMapping property) {
pathStack.pop();
}
}
@@ -0,0 +1,130 @@
package com.avaje.ebeanservice.docstore.api.mapping;
import com.avaje.ebean.annotation.DocMapping;
import java.util.ArrayList;
import java.util.List;
/**
* Property mapping in a doc store document structure.
*/
public class DocPropertyMapping {
private String name;
private DocPropertyType type;
private DocPropertyOptions options;
private List<DocPropertyMapping> children = new ArrayList<DocPropertyMapping>();
/**
* Construct ROOT.
*/
public DocPropertyMapping() {
this.type = DocPropertyType.ROOT;
}
/**
* Construct property mapping.
*/
public DocPropertyMapping(String name, DocPropertyType type) {
this.type = type;
this.name = name;
this.options = new DocPropertyOptions();
}
/**
* Construct property mapping with options.
*/
public DocPropertyMapping(String name, DocPropertyType type, DocPropertyOptions options) {
this.name = name;
this.type = type;
this.options = options;
}
/**
* Visit this property and any nested children.
*/
public void visit(DocPropertyVisitor visitor) {
switch (type) {
case ROOT:
visitor.visitBegin();
visitChildren(visitor);
visitor.visitEnd();
break;
case OBJECT:
visitor.visitBeginObject(this);
visitChildren(visitor);
visitor.visitEndObject(this);
break;
case LIST:
visitor.visitBeginList(this);
visitChildren(visitor);
visitor.visitEndList(this);
break;
default:
visitor.visitProperty(this);
}
}
private void visitChildren(DocPropertyVisitor visitor) {
for (DocPropertyMapping property : children) {
property.visit(visitor);
}
}
public String toString() {
return "name:"+name+" type:"+type+" options("+options+")";
}
/**
* Return the type of the property.
*/
public DocPropertyType getType() {
return type;
}
/**
* Set the type of the property.
*/
public void setType(DocPropertyType type) {
this.type = type;
}
/**
* Return the property name.
*/
public String getName() {
return name;
}
/**
* Return the property options.
*/
public DocPropertyOptions getOptions() {
return options;
}
/**
* Return the child nested properties.
*/
public List<DocPropertyMapping> getChildren() {
return children;
}
/**
* Add a child property.
*/
public void addChild(DocPropertyMapping docMapping) {
children.add(docMapping);
}
/**
* Apply mapping options to this property.
*/
public void apply(DocMapping docMapping) {
options.apply(docMapping);
}
}
@@ -0,0 +1,138 @@
package com.avaje.ebeanservice.docstore.api.mapping;
import com.avaje.ebean.annotation.DocMapping;
/**
* Options for mapping a property for document storage.
*/
public class DocPropertyOptions {
private Boolean code;
private Boolean sortable;
private Boolean store;
private Float boost;
private String nullValue;
/**
* Construct with no values set.
*/
public DocPropertyOptions() {
}
/**
* Construct as a copy of the source options.
*/
protected DocPropertyOptions(DocPropertyOptions source) {
this.code = source.code;
this.sortable = source.sortable;
this.store = source.store;
this.boost = source.boost;
this.nullValue = source.nullValue;
}
/**
* Construct with options set.
*/
public DocPropertyOptions(Boolean code, Boolean sortable, Boolean store, Float boost, String nullValue) {
this.code = code;
this.sortable = sortable;
this.store = store;
this.boost = boost;
this.nullValue = nullValue;
}
public String toString() {
StringBuilder sb = new StringBuilder();
if (code != null) {
sb.append("code:").append(code).append(" ");
}
if (sortable != null) {
sb.append("sortable:").append(sortable).append(" ");
}
if (store != null) {
sb.append("store:").append(store).append(" ");
}
if (boost != null) {
sb.append("boost:").append(boost).append(" ");
}
if (nullValue != null) {
sb.append("nullValue:").append(nullValue).append(" ");
}
return sb.toString();
}
public Boolean getCode() {
return code;
}
public void setCode(Boolean code) {
this.code = code;
}
public Boolean getSortable() {
return sortable;
}
public void setSortable(Boolean sortable) {
this.sortable = sortable;
}
public Float getBoost() {
return boost;
}
public void setBoost(Float boost) {
this.boost = boost;
}
public String getNullValue() {
return nullValue;
}
public void setNullValue(String nullValue) {
this.nullValue = nullValue;
}
public Boolean getStore() {
return store;
}
public void setStore(Boolean store) {
this.store = store;
}
/**
* Create a copy of this such that it can be overridden on a per index basis.
*/
public DocPropertyOptions copy() {
return new DocPropertyOptions(this);
}
/**
* Apply override mapping from the document level or embedded property level.
*/
public void apply(DocMapping docMapping) {
if (docMapping.code()) {
code = true;
}
if (docMapping.sortable()) {
sortable = true;
}
if (docMapping.store()) {
store = true;
}
if (docMapping.boost() != 1) {
boost = docMapping.boost();
}
if (!"".equals(docMapping.nullValue())) {
nullValue = docMapping.nullValue();
}
}
}
@@ -0,0 +1,78 @@
package com.avaje.ebeanservice.docstore.api.mapping;
/**
* Types as defined for document store property types.
*/
public enum DocPropertyType {
/**
* Enum.
*/
ENUM,
/**
* String.
*/
STRING,
/**
* Boolean.
*/
BOOLEAN,
/**
* Short.
*/
SHORT,
/**
* Integer.
*/
INTEGER,
/**
* Long.
*/
LONG,
/**
* Float.
*/
FLOAT,
/**
* Double.
*/
DOUBLE,
/**
* Date without time.
*/
DATE,
/**
* Date with time.
*/
DATETIME,
/**
* Binary type.
*/
BINARY,
/**
* A nested object.
*/
OBJECT,
/**
* A nested list of objects.
*/
LIST,
/**
* Root level type.
*/
ROOT
}
@@ -0,0 +1,43 @@
package com.avaje.ebeanservice.docstore.api.mapping;
/**
* Used to visit the properties in a document structure.
*/
public interface DocPropertyVisitor {
/**
* Begin visiting the document structure.
*/
void visitBegin();
/**
* Visit a property.
*/
void visitProperty(DocPropertyMapping property);
/**
* Start visiting a nested object.
*/
void visitBeginObject(DocPropertyMapping property);
/**
* End visiting a nested object.
*/
void visitEndObject(DocPropertyMapping property);
/**
* Start visiting a nested list.
*/
void visitBeginList(DocPropertyMapping property);
/**
* End visiting a nested list.
*/
void visitEndList(DocPropertyMapping property);
/**
* Finished visiting the document structure.
*/
void visitEnd();
}
@@ -0,0 +1,103 @@
package com.avaje.ebeanservice.docstore.api.mapping;
import com.avaje.ebean.FetchPath;
/**
* Mapping for a document stored in a doc store (like ElasticSearch).
*/
public class DocumentMapping {
protected final String queueId;
protected final String name;
protected final String type;
protected final FetchPath paths;
protected final DocPropertyMapping properties;
protected int shards;
protected int replicas;
public DocumentMapping(String queueId, String name, String type, FetchPath paths, DocPropertyMapping properties, int shards, int replicas) {
this.queueId = queueId;
this.name = name;
this.type = type;
this.paths = paths;
this.properties = properties;
this.shards = shards;
this.replicas = replicas;
}
/**
* Visit all the properties in the document structure.
*/
public void visit(DocPropertyVisitor visitor) {
properties.visit(visitor);
}
/**
* Return the queueId.
*/
public String getQueueId() {
return queueId;
}
/**
* Return the name.
*/
public String getName() {
return name;
}
/**
* Return the type.
*/
public String getType() {
return type;
}
/**
* Return the document structure as PathProperties.
*/
public FetchPath getPaths() {
return paths;
}
/**
* Return the document structure with mapping details.
*/
public DocPropertyMapping getProperties() {
return properties;
}
/**
* Return the number of shards.
*/
public int getShards() {
return shards;
}
/**
* Set the number of shards.
*/
public void setShards(int shards) {
this.shards = shards;
}
/**
* Return the number of replicas.
*/
public int getReplicas() {
return replicas;
}
/**
* Set the number of replicas.
*/
public void setReplicas(int replicas) {
this.replicas = replicas;
}
}
@@ -0,0 +1,315 @@
package com.avaje.ebeanservice.docstore.api.support;
import com.avaje.ebean.FetchPath;
import com.avaje.ebean.Query;
import com.avaje.ebean.annotation.DocStore;
import com.avaje.ebean.annotation.DocStoreEvent;
import com.avaje.ebean.text.PathProperties;
import com.avaje.ebeaninternal.api.SpiEbeanServer;
import com.avaje.ebeaninternal.server.core.PersistRequest;
import com.avaje.ebeaninternal.server.core.PersistRequestBean;
import com.avaje.ebeaninternal.server.deploy.BeanDescriptor;
import com.avaje.ebeaninternal.server.deploy.BeanProperty;
import com.avaje.ebeaninternal.server.deploy.meta.DeployBeanDescriptor;
import com.avaje.ebeanservice.docstore.api.DocStoreBeanAdapter;
import com.avaje.ebeanservice.docstore.api.DocStoreUpdateContext;
import com.avaje.ebeanservice.docstore.api.DocStoreUpdates;
import com.avaje.ebeanservice.docstore.api.mapping.DocMappingBuilder;
import com.avaje.ebeanservice.docstore.api.mapping.DocumentMapping;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Base implementation for much of DocStoreBeanAdapter.
*/
public abstract class DocStoreBeanBaseAdapter<T> implements DocStoreBeanAdapter<T> {
protected final SpiEbeanServer server;
/**
* The associated BeanDescriptor.
*/
protected final BeanDescriptor<T> desc;
/**
* The type of index.
*/
protected final boolean mapped;
/**
* Nested path properties defining the doc structure for indexing.
*/
protected final DocStructure docStructure;
/**
* Identifier used in the queue system to identify the index.
*/
protected final String queueId;
/**
* ElasticSearch index type.
*/
protected final String indexType;
/**
* ElasticSearch index name.
*/
protected final String indexName;
/**
* Doc store deployment annotation.
*/
private final DocStore docStore;
/**
* Behavior on insert.
*/
protected final DocStoreEvent insert;
/**
* Behavior on update.
*/
protected final DocStoreEvent update;
/**
* Behavior on delete.
*/
protected final DocStoreEvent delete;
/**
* List of embedded paths from other documents that include this document type.
* As such an update to this doc type means that those embedded documents need to be updated.
*/
protected final List<DocStoreEmbeddedInvalidation> embeddedInvalidation = new ArrayList<DocStoreEmbeddedInvalidation>();
/**
* Map of properties to 'raw' properties.
*/
private Map<String, String> sortableMap;
public DocStoreBeanBaseAdapter(BeanDescriptor<T> desc, DeployBeanDescriptor<T> deploy) {
this.desc = desc;
this.server = desc.getEbeanServer();
this.mapped = deploy.isDocStoreMapped();
this.docStructure = (!mapped) ? null : derivePathProperties(deploy);
this.docStore = deploy.getDocStore();
this.queueId = derive(desc, deploy.getDocStoreQueueId());
this.indexName = derive(desc, deploy.getDocStoreIndexName());
this.indexType = derive(desc, deploy.getDocStoreIndexType());
this.insert = deploy.getDocStoreInsertEvent();
this.update = deploy.getDocStoreUpdateEvent();
this.delete = deploy.getDocStoreDeleteEvent();
}
@Override
public DocumentMapping createDocMapping() {
if (!mapped) return null;
PathProperties paths = docStructure.doc();
DocMappingBuilder mappingBuilder = new DocMappingBuilder(paths, docStore);
desc.docStoreMapping(mappingBuilder, null);
mappingBuilder.applyMapping();
prepareMapping(mappingBuilder);
sortableMap = mappingBuilder.collectSortable();
docStructure.prepareMany(desc);
return mappingBuilder.create(queueId, indexName, indexType);
}
protected void prepareMapping(DocMappingBuilder mappingBuilder) {
// do nothing by default
}
@Override
public String getIndexType() {
return indexType;
}
@Override
public String getIndexName() {
return indexName;
}
@Override
public void applyPath(Query<T> query) {
query.apply(docStructure.doc());
}
@Override
public String rawProperty(String property) {
String rawProperty = sortableMap.get(property);
return rawProperty == null ? property : rawProperty;
}
/**
* Register invalidation paths for embedded documents.
*/
@Override
public void registerPaths() {
if (mapped) {
Collection<PathProperties.Props> pathProps = docStructure.doc().getPathProps();
for (PathProperties.Props pathProp : pathProps) {
String path = pathProp.getPath();
if (path != null) {
BeanDescriptor<?> targetDesc = desc.getBeanDescriptor(path);
String idName = targetDesc.getIdProperty().getName();
String fullPath = path + "." + idName;
targetDesc.docStoreAdapter().registerInvalidationPath(desc.getDocStoreQueueId(), fullPath, pathProp.getProperties());
}
}
}
}
/**
* Register a doc store invalidation listener for the given bean type, path and properties.
*/
@Override
public void registerInvalidationPath(String queueId, String path, Set<String> properties) {
embeddedInvalidation.add(getEmbeddedInvalidation(queueId, path, properties));
}
/**
* Return the DsInvalidationListener based on the properties, path.
*/
protected DocStoreEmbeddedInvalidation getEmbeddedInvalidation(String queueId, String path, Set<String> properties) {
if (properties.contains("*")) {
return new DocStoreEmbeddedInvalidation(queueId, path);
} else {
return new DocStoreEmbeddedInvalidationProperties(queueId, path, getPropertyPositions(properties));
}
}
/**
* Return the property names as property index positions.
*/
protected int[] getPropertyPositions(Set<String> properties) {
List<Integer> posList = new ArrayList<Integer>();
for (String property : properties) {
BeanProperty prop = desc.getBeanProperty(property);
if (prop != null) {
posList.add(prop.getPropertyIndex());
}
}
int[] pos = new int[posList.size()];
for (int i = 0; i <pos.length; i++) {
pos[i] = posList.get(i);
}
return pos;
}
@Override
public void updateEmbedded(PersistRequestBean<T> request, DocStoreUpdates docStoreUpdates) {
for (int i = 0; i < embeddedInvalidation.size(); i++) {
embeddedInvalidation.get(i).embeddedInvalidate(request, docStoreUpdates);
}
}
/**
* Return the pathProperties which defines the JSON document to index.
* This can add derived/embedded/nested parts to the document.
*/
protected DocStructure derivePathProperties(DeployBeanDescriptor<T> deploy) {
if (!mapped) {
return null;
}
PathProperties pathProps = deploy.getDocStorePathProperties();
boolean includeByDefault = (pathProps == null);
if (pathProps == null) {
pathProps = new PathProperties();
}
return getDocStructure(pathProps, includeByDefault);
}
protected DocStructure getDocStructure(PathProperties pathProps, boolean includeByDefault) {
DocStructure docStructure = new DocStructure(pathProps);
BeanProperty[] properties = desc.propertiesNonTransient();
for (int i = 0; i < properties.length; i++) {
properties[i].docStoreInclude(includeByDefault, docStructure);
}
return docStructure;
}
public FetchPath getEmbedded(String path) {
return docStructure.getEmbedded(path);
}
public FetchPath getEmbeddedManyRoot(String path) {
return docStructure.getEmbeddedManyRoot(path);
}
@Override
public boolean isMapped() {
return mapped;
}
@Override
public String getQueueId() {
return queueId;
}
@Override
public DocStoreEvent getEvent(PersistRequest.Type persistType, DocStoreEvent txnMode) {
if (txnMode == null) {
return getDocStoreEvent(persistType);
} else if (txnMode == DocStoreEvent.IGNORE) {
return DocStoreEvent.IGNORE;
}
return mapped ? txnMode : getDocStoreEvent(persistType);
}
private DocStoreEvent getDocStoreEvent(PersistRequest.Type persistType) {
switch (persistType) {
case INSERT:
return insert;
case UPDATE:
return update;
case DELETE:
return delete;
default:
return DocStoreEvent.IGNORE;
}
}
/**
* Return the supplied value or default to the bean name lower case.
*/
protected String derive(BeanDescriptor<T> desc, String suppliedValue) {
return (suppliedValue != null && suppliedValue.length() > 0) ? suppliedValue : desc.getName().toLowerCase();
}
@Override
public abstract void deleteById(Object idValue, DocStoreUpdateContext txn) throws IOException;
@Override
public abstract void index(Object idValue, T entityBean, DocStoreUpdateContext txn) throws IOException;
@Override
public abstract void insert(Object idValue, PersistRequestBean<T> persistRequest, DocStoreUpdateContext txn) throws IOException;
@Override
public abstract void update(Object idValue, PersistRequestBean<T> persistRequest, DocStoreUpdateContext txn) throws IOException;
@Override
public abstract void updateEmbedded(Object idValue, String embeddedProperty, String embeddedRawContent, DocStoreUpdateContext txn) throws IOException;
}
@@ -0,0 +1,39 @@
package com.avaje.ebeanservice.docstore.api.support;
import com.avaje.ebean.plugin.BeanType;
import com.avaje.ebeanservice.docstore.api.DocStoreUpdateContext;
import com.avaje.ebeanservice.docstore.api.DocStoreUpdate;
import com.avaje.ebeanservice.docstore.api.DocStoreUpdates;
import java.io.IOException;
/**
* A 'Delete by Id' request that is send to the document store.
*/
public class DocStoreDeleteEvent implements DocStoreUpdate {
private final BeanType<?> beanType;
private final Object idValue;
public DocStoreDeleteEvent(BeanType<?> beanType, Object idValue) {
this.beanType = beanType;
this.idValue = idValue;
}
/**
* Add appropriate JSON content for sending to the ElasticSearch Bulk API.
*/
@Override
public void docStoreUpdate(DocStoreUpdateContext txn) throws IOException {
beanType.docStore().deleteById(idValue, txn);
}
/**
* Add this event to the queue (for queue delayed processing).
*/
@Override
public void addToQueue(DocStoreUpdates docStoreUpdates) {
docStoreUpdates.queueDelete(beanType.getDocStoreQueueId(), idValue);
}
}
@@ -0,0 +1,23 @@
package com.avaje.ebeanservice.docstore.api.support;
import com.avaje.ebeaninternal.server.core.PersistRequestBean;
import com.avaje.ebeanservice.docstore.api.DocStoreUpdates;
/**
* Checks if a persist request means an embedded/nested object in another document needs updating.
*/
public class DocStoreEmbeddedInvalidation {
protected final String queueId;
protected final String path;
public DocStoreEmbeddedInvalidation(String queueId, String path) {
this.queueId = queueId;
this.path = path;
}
public void embeddedInvalidate(PersistRequestBean<?> request, DocStoreUpdates docStoreUpdates) {
docStoreUpdates.addNested(queueId, path, request.getBeanId());
}
}
@@ -0,0 +1,30 @@
package com.avaje.ebeanservice.docstore.api.support;
import com.avaje.ebeaninternal.server.core.PersistRequestBean;
import com.avaje.ebeanservice.docstore.api.DocStoreUpdates;
/**
* Checks if a persist request means an embedded/nested object in another document needs updating.
*
* This has specific properties to check (so not all properties invalidate).
*/
public final class DocStoreEmbeddedInvalidationProperties extends DocStoreEmbeddedInvalidation {
/**
* Properties that trigger invalidation.
*/
private final int[] properties;
public DocStoreEmbeddedInvalidationProperties(String queueId, String path, int[] properties) {
super(queueId, path);
this.properties = properties;
}
@Override
public void embeddedInvalidate(PersistRequestBean<?> request, DocStoreUpdates docStoreUpdates) {
if (request.hasDirtyProperty(properties)) {
docStoreUpdates.addNested(queueId, path, request.getBeanId());
}
}
}
@@ -0,0 +1,42 @@
package com.avaje.ebeanservice.docstore.api.support;
import com.avaje.ebean.plugin.BeanType;
import com.avaje.ebeanservice.docstore.api.DocStoreUpdateContext;
import com.avaje.ebeanservice.docstore.api.DocStoreUpdate;
import com.avaje.ebeanservice.docstore.api.DocStoreUpdates;
import java.io.IOException;
/**
* A 'Delete by Id' request that is send to the document store.
*/
public class DocStoreIndexEvent<T> implements DocStoreUpdate {
private final BeanType<T> beanType;
private final Object idValue;
private final T bean;
public DocStoreIndexEvent(BeanType<T> beanType, Object idValue, T bean) {
this.beanType = beanType;
this.idValue = idValue;
this.bean = bean;
}
/**
* Add appropriate JSON content for sending to the ElasticSearch Bulk API.
*/
@Override
public void docStoreUpdate(DocStoreUpdateContext txn) throws IOException {
beanType.docStore().index(idValue, bean, txn);
}
/**
* Add this event to the queue (for queue delayed processing).
*/
@Override
public void addToQueue(DocStoreUpdates docStoreUpdates) {
docStoreUpdates.queueIndex(beanType.getDocStoreQueueId(), idValue);
}
}
@@ -0,0 +1,96 @@
package com.avaje.ebeanservice.docstore.api.support;
import com.avaje.ebean.text.PathProperties;
import com.avaje.ebean.FetchPath;
import com.avaje.ebeaninternal.server.deploy.BeanDescriptor;
import com.avaje.ebeaninternal.server.deploy.BeanProperty;
import com.avaje.ebeaninternal.server.deploy.BeanPropertyAssoc;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
/**
* Document structure for mapping to document store.
*/
public class DocStructure {
/**
* The full document structure.
*/
private final PathProperties doc;
/**
* The embedded document structures by path.
*/
private final Map<String,PathProperties> embedded = new HashMap<String, PathProperties>();
private final Map<String,PathProperties> manyRoot = new HashMap<String, PathProperties>();
/**
* Create given an initial deployment doc mapping.
*/
public DocStructure(PathProperties pathProps) {
this.doc = pathProps;
}
/**
* Add a property at the root level.
*/
public void addProperty(String name) {
doc.addToPath(null, name);
}
/**
* Add an embedded property with it's document structure.
*/
public void addNested(String path, PathProperties embeddedDoc) {
doc.addNested(path, embeddedDoc);
embedded.put(path, embeddedDoc);
}
/**
* Return the document structure.
*/
public PathProperties doc() {
return doc;
}
/**
* Return the document structure for an embedded path.
*/
public FetchPath getEmbedded(String path) {
return embedded.get(path);
}
public FetchPath getEmbeddedManyRoot(String path) {
return manyRoot.get(path);
}
/**
* For 'many' nested properties we need an additional root based graph to fetch and update.
*/
public <T> void prepareMany(BeanDescriptor<T> desc) {
Set<String> strings = embedded.keySet();
for (String prop : strings) {
BeanPropertyAssoc<?> embProp = (BeanPropertyAssoc<?>)desc.getBeanProperty(prop);
if (embProp.isMany()) {
prepare(prop, embProp);
}
}
}
/**
* Add a PathProperties for an embedded 'many' property (at the root level).
*/
private void prepare(String prop, BeanPropertyAssoc<?> embProp) {
BeanDescriptor<?> targetDesc = embProp.getTargetDescriptor();
PathProperties manyRootPath = new PathProperties();
manyRootPath.addToPath(null, targetDesc.getIdProperty().getName());
manyRootPath.addNested(prop, embedded.get(prop));
manyRoot.put(prop, manyRootPath);
}
}