mirror of
https://github.com/ebean-orm/ebean.git
synced 2026-11-18 14:58:47 +00:00
#390 - ENH: Add Change log mechanism for easy fine grained logging of insert, update, delete activity.
This adds default implementations for register,filter, prepare and log of the change sets.
This commit is contained in:
1 parent
31099f6dda
commit
fe8a32ea18
25 files changed
+764
-19
No files matched your search
@@ -0,0 +1,173 @@
|
||||
package com.avaje.ebeaninternal.server.changelog;
|
||||
|
||||
import com.avaje.ebean.ValuePair;
|
||||
import com.avaje.ebean.event.changelog.BeanChange;
|
||||
import com.avaje.ebean.event.changelog.ChangeSet;
|
||||
import com.avaje.ebean.event.changelog.ChangeType;
|
||||
import com.avaje.ebean.text.json.JsonContext;
|
||||
import com.avaje.ebean.text.json.JsonScalar;
|
||||
import com.fasterxml.jackson.core.JsonFactory;
|
||||
import com.fasterxml.jackson.core.JsonGenerator;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Writer;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Builds JSON appropriate for loading into ElasticS via the bulk API.
|
||||
*/
|
||||
public class BulkJsonBuilder {
|
||||
|
||||
protected final JsonFactory jsonFactory = new JsonFactory();
|
||||
|
||||
protected final String indexName;
|
||||
|
||||
protected final String indexType;
|
||||
|
||||
protected final JsonContext json;
|
||||
|
||||
protected BulkJsonBuilder(JsonContext json, String indexName, String indexType) {
|
||||
this.json = json;
|
||||
this.indexName = indexName;
|
||||
this.indexType = indexType;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the change set into Elastic bulk API JSON form (so contains special new line
|
||||
* characters and bulk API header etc.
|
||||
*/
|
||||
public void writeJson(ChangeSet changeSet, Writer writer) throws IOException {
|
||||
|
||||
JsonGenerator generator = jsonFactory.createGenerator(writer);
|
||||
|
||||
List<BeanChange> changes = changeSet.getChanges();
|
||||
for (int i = 0; i < changes.size(); i++) {
|
||||
write(generator, changes.get(i), changeSet, i);
|
||||
}
|
||||
|
||||
generator.flush();
|
||||
generator.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the bean change as a single JSON document for storage into Elastic.
|
||||
* <p>
|
||||
* Note that for ease of search/use we effectively denormalise by including the transaction header
|
||||
* information in each bean document.
|
||||
* </p>
|
||||
*/
|
||||
protected void write(JsonGenerator gen, BeanChange bean, ChangeSet changeSet, int position) throws IOException {
|
||||
|
||||
writeBulkHeader(gen, changeSet, position);
|
||||
writeBeanChange(gen, bean, changeSet);
|
||||
writeBeanChangeEnd(gen);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the bean change as JSON document containing the transaction header details.
|
||||
*/
|
||||
protected void writeBeanChange(JsonGenerator gen, BeanChange bean, ChangeSet changeSet) throws IOException {
|
||||
|
||||
gen.writeStartObject();
|
||||
|
||||
writeBeanTransactionDetails(gen, changeSet);
|
||||
|
||||
gen.writeStringField("object", bean.getTable());
|
||||
gen.writeStringField("objectId", bean.getId().toString());
|
||||
gen.writeStringField("change", bean.getType().getCode());
|
||||
gen.writeNumberField("eventTime", bean.getEventTime());
|
||||
|
||||
writeBeanValues(gen, bean);
|
||||
|
||||
gen.writeEndObject();
|
||||
}
|
||||
|
||||
/**
|
||||
* For Elastic bulk we append raw new line character.
|
||||
*/
|
||||
protected void writeBeanChangeEnd(JsonGenerator gen) throws IOException {
|
||||
gen.writeRawValue("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Denormalise by writing the transaction header details.
|
||||
*/
|
||||
protected void writeBeanTransactionDetails(JsonGenerator gen, ChangeSet changeSet) throws IOException {
|
||||
|
||||
gen.writeStringField("txnId", changeSet.getTxnId());
|
||||
gen.writeStringField("txnState", changeSet.getTxnState().getCode());
|
||||
gen.writeNumberField("txnBatch", changeSet.getTxnBatch());
|
||||
gen.writeStringField("userId", changeSet.getUserId());
|
||||
String userIpAddress = changeSet.getUserIpAddress();
|
||||
if (userIpAddress != null) {
|
||||
gen.writeStringField("userIpAddress", userIpAddress);
|
||||
}
|
||||
String userContext = changeSet.getUserContext();
|
||||
if (userContext != null) {
|
||||
gen.writeStringField("userContext", userContext);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the elastic bulk API header.
|
||||
*/
|
||||
protected void writeBulkHeader(JsonGenerator gen, ChangeSet changeSet, int position) throws IOException {
|
||||
|
||||
// we index with an 'id' value so that we can process/reprocess the JSON and
|
||||
// avoid duplicates being inserted. Appending the batch and position give us
|
||||
// an effectively unique id value for the change
|
||||
String uid = changeSet.getTxnId() + "_" + changeSet.getTxnBatch() + "." + position;
|
||||
|
||||
// the 'header' for elastic bulk API
|
||||
gen.writeStartObject();
|
||||
gen.writeFieldName("index");
|
||||
gen.writeStartObject();
|
||||
gen.writeStringField("_index", indexName);
|
||||
gen.writeStringField("_type", indexType);
|
||||
gen.writeStringField("_id", uid);
|
||||
gen.writeEndObject();
|
||||
gen.writeEndObject();
|
||||
gen.writeRawValue("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* For insert and update write the new/old values.
|
||||
*/
|
||||
protected void writeBeanValues(JsonGenerator gen, BeanChange bean) throws IOException {
|
||||
if (bean.getType() != ChangeType.DELETE) {
|
||||
gen.writeFieldName("values");
|
||||
gen.writeStartObject();
|
||||
// use JsonScalar as it knows how to encode all the scalar
|
||||
// property types that Ebean supports (Java8, Joda etc)
|
||||
JsonScalar scalarWriter = json.getScalar(gen);
|
||||
writeValuePairs(bean, scalarWriter, gen);
|
||||
gen.writeEndObject();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write all the value pairs suppressing null values.
|
||||
* <p>
|
||||
* We are intentionally keeping the same new/old structure for both inserts and updates.
|
||||
* </p>
|
||||
*/
|
||||
protected void writeValuePairs(BeanChange bean, JsonScalar scalarWriter, JsonGenerator gen) throws IOException {
|
||||
|
||||
for (Map.Entry<String, ValuePair> entry : bean.getValues().entrySet()) {
|
||||
gen.writeFieldName(entry.getKey());
|
||||
gen.writeStartObject();
|
||||
ValuePair value = entry.getValue();
|
||||
Object newValue = value.getNewValue();
|
||||
if (newValue != null) {
|
||||
scalarWriter.write("new", newValue);
|
||||
}
|
||||
Object oldValue = value.getOldValue();
|
||||
if (oldValue != null) {
|
||||
scalarWriter.write("old", oldValue);
|
||||
}
|
||||
gen.writeEndObject();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.avaje.ebeaninternal.server.changelog;
|
||||
|
||||
import com.avaje.ebean.event.changelog.ChangeLogPrepare;
|
||||
import com.avaje.ebean.event.changelog.ChangeSet;
|
||||
|
||||
/**
|
||||
* Placeholder/default implementation that does not do anything.
|
||||
* <p>
|
||||
* Generally an implementation should be provided that reads context
|
||||
* information such as user id and user ip address etc and sets that
|
||||
* on the changeSet.
|
||||
* </p>
|
||||
*/
|
||||
public class DefaultChangeLogPrepare implements ChangeLogPrepare {
|
||||
|
||||
/**
|
||||
* Just return true to send change set through to the logger.
|
||||
*/
|
||||
@Override
|
||||
public boolean prepare(ChangeSet changeSet) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package com.avaje.ebeaninternal.server.changelog;
|
||||
|
||||
import com.avaje.ebean.annotation.ChangeLog;
|
||||
import com.avaje.ebean.event.BeanPersistRequest;
|
||||
import com.avaje.ebean.event.changelog.ChangeLogFilter;
|
||||
import com.avaje.ebean.event.changelog.ChangeLogRegister;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Default implementation of ChangeLogRegister.
|
||||
*/
|
||||
public class DefaultChangeLogRegister implements ChangeLogRegister {
|
||||
|
||||
private static final BasicFilter INCLUDE_INSERTS = new BasicFilter(true);
|
||||
|
||||
private static final BasicFilter EXCLUDE_INSERTS = new BasicFilter(false);
|
||||
|
||||
|
||||
@Override
|
||||
public ChangeLogFilter getChangeFilter(Class<?> beanType) {
|
||||
|
||||
ChangeLog changeLog = beanType.getAnnotation(ChangeLog.class);
|
||||
if (changeLog == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String[] updatesThatInclude = changeLog.updatesThatInclude();
|
||||
if (updatesThatInclude.length == 0) {
|
||||
return changeLog.excludeInserts() ? EXCLUDE_INSERTS : INCLUDE_INSERTS;
|
||||
}
|
||||
|
||||
Set<String> updateProps = new HashSet<String>();
|
||||
for (int i = 0; i < updatesThatInclude.length; i++) {
|
||||
updateProps.add(updatesThatInclude[i]);
|
||||
}
|
||||
|
||||
return new UpdateFilter(!changeLog.excludeInserts(), updateProps);
|
||||
}
|
||||
|
||||
/**
|
||||
* Basic filter that only handles include inserts flag.
|
||||
*/
|
||||
protected static class BasicFilter implements ChangeLogFilter {
|
||||
|
||||
final boolean includeInserts;
|
||||
|
||||
BasicFilter(boolean includeInserts) {
|
||||
this.includeInserts = includeInserts;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean includeInsert(BeanPersistRequest<?> insertRequest) {
|
||||
return includeInserts;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean includeUpdate(BeanPersistRequest<?> updateRequest) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean includeDelete(BeanPersistRequest<?> deleteRequest) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter that takes into account a set of properties to check for updates
|
||||
* as well as the include inserts flag.
|
||||
*/
|
||||
protected static class UpdateFilter extends BasicFilter {
|
||||
|
||||
final Set<String> updateProperties;
|
||||
|
||||
UpdateFilter(boolean includeInserts, Set<String> updateProperties) {
|
||||
super(includeInserts);
|
||||
this.updateProperties = updateProperties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean includeUpdate(BeanPersistRequest<?> updateRequest) {
|
||||
return updateRequest.hasDirtyProperty(updateProperties);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package com.avaje.ebeaninternal.server.changelog;
|
||||
|
||||
import com.avaje.ebean.event.changelog.ChangeLogListener;
|
||||
import com.avaje.ebean.event.changelog.ChangeSet;
|
||||
import com.avaje.ebean.plugin.SpiServer;
|
||||
import com.avaje.ebean.plugin.SpiServerPlugin;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
|
||||
/**
|
||||
* Logs the change sets in a Elastic Search Bulk API JSON format.
|
||||
*/
|
||||
public class ElasticChangeLogListener implements ChangeLogListener, SpiServerPlugin {
|
||||
|
||||
protected static final Logger appLog = LoggerFactory.getLogger(ElasticChangeLogListener.class);
|
||||
|
||||
protected static final Logger fileLog = LoggerFactory.getLogger("org.avaje.ebean.ElasticChangeLog");
|
||||
|
||||
protected BulkJsonBuilder jsonBuilder;
|
||||
|
||||
public ElasticChangeLogListener() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Configure the underlying JSON handler.
|
||||
*/
|
||||
@Override
|
||||
public void configure(SpiServer server) {
|
||||
jsonBuilder = new BulkJsonBuilder(server.json(), "changelog", "changelog");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void online(boolean online) {
|
||||
// We don't care online or offline in this case
|
||||
// we might if we setup for network sending etc
|
||||
}
|
||||
|
||||
@Override
|
||||
public void log(ChangeSet changeSet) {
|
||||
|
||||
try {
|
||||
// I'm pretty sure I'm going to change this to use a FileWriter (but apache kafta could be good here too)
|
||||
// This buffer could get really big and to me normal logging (without a Writer) is not that well suited to
|
||||
// this problem so ... works but lets do better here and write direct to files (without the buffer issue)
|
||||
StringWriter writer = new StringWriter(getBufferSize(changeSet));
|
||||
jsonBuilder.writeJson(changeSet, writer);
|
||||
String json = writer.toString();
|
||||
|
||||
fileLog.info("Sending txnId:{} txnState:{} txnBatch:{} \n {}", changeSet.getTxnId(), changeSet.getTxnState(), changeSet.getTxnBatch(), json);
|
||||
|
||||
} catch (IOException e) {
|
||||
String msg = extractErrorMessage(e);
|
||||
fileLog.error("Exception sending txnId:{} txnState:{} txnBatch:{} error:{}", changeSet.getTxnId(), changeSet.getTxnState(), changeSet.getTxnBatch(), msg);
|
||||
appLog.error("Exception sending changeSet "+changeSet.toString(), e);
|
||||
}
|
||||
}
|
||||
|
||||
protected int getBufferSize(ChangeSet changeSet) {
|
||||
// a rough guess, could get smarter here
|
||||
return Math.min(400 * changeSet.size(), 3000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract an error message that does not have new line characters and hence safe to go into
|
||||
* our log which contains the payloads (that we want to be able to extract easily later on).
|
||||
*/
|
||||
@NotNull
|
||||
protected String extractErrorMessage(Exception e) {
|
||||
|
||||
String msg = e.toString();
|
||||
msg = msg.replace('\r','|');
|
||||
msg = msg.replace('\n', '|');
|
||||
return msg;
|
||||
}
|
||||
|
||||
}
|
||||
Reference in new issue
Block a user