Extract CsvReader into separate ebean-csv-reader module

This commit is contained in:
Rob Bygrave committed 2022-08-25 17:25:49 +12:00
1 parent 5b2a8cdacb
commit 216c43eea6
22 files changed
+372 -130

No files matched your search

-6
View File
@@ -785,12 +785,6 @@ public final class DB {
return getDefault().createUpdate(beanType, ormUpdate); return getDefault().createUpdate(beanType, ormUpdate);
} }
/**
* Create a CsvReader for a given beanType.
*/
public static <T> CsvReader<T> createCsvReader(Class<T> beanType) {
return getDefault().createCsvReader(beanType);
}
/** /**
* Create a named query. * Create a named query.
@@ -204,10 +204,6 @@ public interface Database {
*/ */
<T> T createEntityBean(Class<T> type); <T> T createEntityBean(Class<T> type);
/**
* Create a CsvReader for a given beanType.
*/
<T> CsvReader<T> createCsvReader(Class<T> beanType);
/** /**
* Create an Update query to perform a bulk update. * Create an Update query to perform a bulk update.
@@ -49,6 +49,7 @@ public class DefaultCsvCallback<T> implements CsvCallback<T> {
*/ */
protected final int persistBatchSize; protected final int persistBatchSize;
protected boolean getGeneratedKeys = true;
/** /**
* The time the process started. * The time the process started.
*/ */
@@ -164,7 +165,7 @@ public class DefaultCsvCallback<T> implements CsvCallback<T> {
if (persistBatchSize > 1) { if (persistBatchSize > 1) {
transaction.setBatchMode(true); transaction.setBatchMode(true);
transaction.setBatchSize(persistBatchSize); transaction.setBatchSize(persistBatchSize);
transaction.setGetGeneratedKeys(false); transaction.setGetGeneratedKeys(getGeneratedKeys);
} else { } else {
// explicitly turn off JDBC batching in case // explicitly turn off JDBC batching in case
// is has been turned on globally // is has been turned on globally
@@ -39,7 +39,6 @@ import io.ebeaninternal.server.grammer.EqlParser;
import io.ebeaninternal.server.query.*; import io.ebeaninternal.server.query.*;
import io.ebeaninternal.server.querydefn.*; import io.ebeaninternal.server.querydefn.*;
import io.ebeaninternal.server.rawsql.SpiRawSql; import io.ebeaninternal.server.rawsql.SpiRawSql;
import io.ebeaninternal.server.text.csv.TCsvReader;
import io.ebeaninternal.server.transaction.DefaultPersistenceContext; import io.ebeaninternal.server.transaction.DefaultPersistenceContext;
import io.ebeaninternal.server.transaction.RemoteTransactionEvent; import io.ebeaninternal.server.transaction.RemoteTransactionEvent;
import io.ebeaninternal.server.transaction.TransactionManager; import io.ebeaninternal.server.transaction.TransactionManager;
@@ -807,10 +806,6 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
return new ElFilter<>(desc(beanType)); return new ElFilter<>(desc(beanType));
} }
@Override
public <T> CsvReader<T> createCsvReader(Class<T> beanType) {
return new TCsvReader<>(this, desc(beanType));
}
@Override @Override
public <T> UpdateQuery<T> update(Class<T> beanType) { public <T> UpdateQuery<T> update(Class<T> beanType) {
@@ -1 +0,0 @@
package io.ebeaninternal.server.text.csv;
+69
View File
@@ -0,0 +1,69 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>ebean-parent</artifactId>
<groupId>io.ebean</groupId>
<version>13.8.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>ebean-csv-reader</artifactId>
<properties>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>io.ebean</groupId>
<artifactId>ebean-api</artifactId>
<version>13.8.2-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.ebean</groupId>
<artifactId>ebean-core</artifactId>
<version>13.8.2-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.ebean</groupId>
<artifactId>ebean-test</artifactId>
<version>13.8.2-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.avaje</groupId>
<artifactId>junit</artifactId>
<version>1.1</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<!-- for testing -->
<plugin>
<groupId>io.repaint.maven</groupId>
<artifactId>tiles-maven-plugin</artifactId>
<version>2.24</version>
<extensions>true</extensions>
<configuration>
<tiles>
<tile>io.ebean.tile:enhancement:13.6.5</tile>
</tiles>
</configuration>
</plugin>
</plugins>
</build>
</project>
@@ -2,16 +2,13 @@ package io.ebeaninternal.server.text.csv;
import io.ebean.Database; import io.ebean.Database;
import io.ebean.bean.EntityBean; import io.ebean.bean.EntityBean;
import io.ebean.plugin.BeanType;
import io.ebean.plugin.ExpressionPath; import io.ebean.plugin.ExpressionPath;
import io.ebean.text.StringParser; import io.ebean.text.StringParser;
import io.ebean.text.TextException; import io.ebean.text.TextException;
import io.ebean.text.TimeStringParser;
import io.ebean.text.csv.CsvCallback; import io.ebean.text.csv.CsvCallback;
import io.ebean.text.csv.CsvReader; import io.ebean.text.csv.CsvReader;
import io.ebean.text.csv.DefaultCsvCallback; import io.ebean.text.csv.DefaultCsvCallback;
import io.ebeaninternal.server.deploy.BeanDescriptor;
import io.ebeaninternal.server.deploy.BeanPropertyAssocOne;
import io.ebeaninternal.server.el.ElPropertyValue;
import java.io.Reader; import java.io.Reader;
import java.sql.Types; import java.sql.Types;
@@ -27,13 +24,13 @@ import java.util.Locale;
/** /**
* Implementation of the CsvReader * Implementation of the CsvReader
*/ */
public class TCsvReader<T> implements CsvReader<T> { public class TCsvReader<T> {//implements CsvReader<T> {
private static final TimeStringParser TIME_PARSER = new TimeStringParser(); private static final TimeStringParser TIME_PARSER = new TimeStringParser();
private final Database server; private final Database server;
private final BeanDescriptor<T> descriptor; private final BeanType<T> descriptor;
private final List<CsvColumn> columnList = new ArrayList<>(); private final List<CsvColumn> columnList = new ArrayList<>();
@@ -55,73 +52,73 @@ public class TCsvReader<T> implements CsvReader<T> {
private boolean addPropertiesFromHeader; private boolean addPropertiesFromHeader;
public TCsvReader(Database server, BeanDescriptor<T> descriptor) { public TCsvReader(Database server, BeanType<T> descriptor) {
this.server = server; this.server = server;
this.descriptor = descriptor; this.descriptor = descriptor;
} }
@Override ////@Override
public void setDefaultLocale(Locale defaultLocale) { public void setDefaultLocale(Locale defaultLocale) {
this.defaultLocale = defaultLocale; this.defaultLocale = defaultLocale;
} }
@Override ////@Override
public void setDefaultTimeFormat(String defaultTimeFormat) { public void setDefaultTimeFormat(String defaultTimeFormat) {
this.defaultTimeFormat = defaultTimeFormat; this.defaultTimeFormat = defaultTimeFormat;
} }
@Override ////@Override
public void setDefaultDateFormat(String defaultDateFormat) { public void setDefaultDateFormat(String defaultDateFormat) {
this.defaultDateFormat = defaultDateFormat; this.defaultDateFormat = defaultDateFormat;
} }
@Override ////@Override
public void setDefaultTimestampFormat(String defaultTimestampFormat) { public void setDefaultTimestampFormat(String defaultTimestampFormat) {
this.defaultTimestampFormat = defaultTimestampFormat; this.defaultTimestampFormat = defaultTimestampFormat;
} }
@Override ////@Override
public void setPersistBatchSize(int persistBatchSize) { public void setPersistBatchSize(int persistBatchSize) {
this.persistBatchSize = persistBatchSize; this.persistBatchSize = persistBatchSize;
} }
@Override ////@Override
public void setIgnoreHeader() { public void setIgnoreHeader() {
setHasHeader(true, false); setHasHeader(true, false);
} }
@Override ////@Override
public void setAddPropertiesFromHeader() { public void setAddPropertiesFromHeader() {
setHasHeader(true, true); setHasHeader(true, true);
} }
@Override //@Override
public void setHasHeader(boolean hasHeader, boolean addPropertiesFromHeader) { public void setHasHeader(boolean hasHeader, boolean addPropertiesFromHeader) {
this.hasHeader = hasHeader; this.hasHeader = hasHeader;
this.addPropertiesFromHeader = addPropertiesFromHeader; this.addPropertiesFromHeader = addPropertiesFromHeader;
} }
@Override //@Override
public void setLogInfoFrequency(int logInfoFrequency) { public void setLogInfoFrequency(int logInfoFrequency) {
this.logInfoFrequency = logInfoFrequency; this.logInfoFrequency = logInfoFrequency;
} }
@Override //@Override
public void addIgnore() { public void addIgnore() {
columnList.add(ignoreColumn); columnList.add(ignoreColumn);
} }
@Override //@Override
public void addProperty(String propertyName) { public void addProperty(String propertyName) {
addProperty(propertyName, null); addProperty(propertyName, null);
} }
@Override //@Override
public void addDateTime(String propertyName, String dateTimeFormat) { public void addDateTime(String propertyName, String dateTimeFormat) {
addDateTime(propertyName, dateTimeFormat, Locale.getDefault()); addDateTime(propertyName, dateTimeFormat, Locale.getDefault());
} }
@Override //@Override
public void addDateTime(String propertyName, String dateTimeFormat, Locale locale) { public void addDateTime(String propertyName, String dateTimeFormat, Locale locale) {
ExpressionPath elProp = descriptor.expressionPath(propertyName); ExpressionPath elProp = descriptor.expressionPath(propertyName);
if (dateTimeFormat == null) { if (dateTimeFormat == null) {
@@ -153,7 +150,7 @@ public class TCsvReader<T> implements CsvReader<T> {
} }
} }
@Override //@Override
public void addProperty(String propertyName, StringParser parser) { public void addProperty(String propertyName, StringParser parser) {
ExpressionPath elProp = descriptor.expressionPath(propertyName); ExpressionPath elProp = descriptor.expressionPath(propertyName);
@@ -164,13 +161,13 @@ public class TCsvReader<T> implements CsvReader<T> {
columnList.add(column); columnList.add(column);
} }
@Override //@Override
public void process(Reader reader) throws Exception { public void process(Reader reader) throws Exception {
DefaultCsvCallback<T> callback = new DefaultCsvCallback<>(persistBatchSize, logInfoFrequency); DefaultCsvCallback<T> callback = new DefaultCsvCallback<>(persistBatchSize, logInfoFrequency);
process(reader, callback); process(reader, callback);
} }
@Override //@Override
public void process(Reader reader, CsvCallback<T> callback) throws Exception { public void process(Reader reader, CsvCallback<T> callback) throws Exception {
if (reader == null) { if (reader == null) {
@@ -231,7 +228,8 @@ public class TCsvReader<T> implements CsvReader<T> {
private void addPropertiesFromHeader(String[] line) { private void addPropertiesFromHeader(String[] line) {
for (String aLine : line) { for (String aLine : line) {
ElPropertyValue elProp = descriptor.elGetValue(aLine); ExpressionPath elProp = descriptor.expressionPath(aLine);
//ElPropertyValue elProp = descriptor.elGetValue(aLine);
if (elProp == null) { if (elProp == null) {
throw new TextException("Property [" + aLine + "] not found"); throw new TextException("Property [" + aLine + "] not found");
} }
@@ -242,10 +240,10 @@ public class TCsvReader<T> implements CsvReader<T> {
} else if (isDateTimeType(elProp.jdbcType())) { } else if (isDateTimeType(elProp.jdbcType())) {
addDateTime(aLine, null, null); addDateTime(aLine, null, null);
} else if (elProp.isAssocProperty()) { // } else if (elProp.isAssocProperty()) {
BeanPropertyAssocOne<?> assocOne = (BeanPropertyAssocOne<?>) elProp.beanProperty(); // BeanPropertyAssocOne<?> assocOne = (BeanPropertyAssocOne<?>) elProp.beanProperty();
String idProp = assocOne.descriptor().idBinder().getIdProperty(); // String idProp = assocOne.descriptor().idBinder().getIdProperty();
addProperty(aLine + "." + idProp); // addProperty(aLine + "." + idProp);
} else { } else {
addProperty(aLine); addProperty(aLine);
} }
@@ -260,8 +258,8 @@ public class TCsvReader<T> implements CsvReader<T> {
protected T buildBeanFromLineContent(int row, String[] line) { protected T buildBeanFromLineContent(int row, String[] line) {
try { try {
EntityBean entityBean = descriptor.createEntityBean(); T bean = descriptor.createBean();
T bean = (T) entityBean; EntityBean entityBean = (EntityBean)bean;
for (int columnPos = 0; columnPos < line.length; columnPos++) { for (int columnPos = 0; columnPos < line.length; columnPos++) {
convertAndSetColumn(columnPos, line[columnPos], entityBean); convertAndSetColumn(columnPos, line[columnPos], entityBean);
@@ -340,7 +338,7 @@ public class TCsvReader<T> implements CsvReader<T> {
this.format = format; this.format = format;
} }
@Override //@Override
public Object parse(String value) { public Object parse(String value) {
try { try {
Date dt = dateFormat.parse(value); Date dt = dateFormat.parse(value);
@@ -1,4 +1,6 @@
package io.ebean.text; package io.ebeaninternal.server.text.csv;
import io.ebean.text.StringParser;
import java.sql.Time; import java.sql.Time;
@@ -1,28 +1,37 @@
package org.tests.text.csv; package io.ebeaninternal.server.text.csv;
import io.ebean.DB; import io.ebean.DB;
import io.ebean.xtest.base.TransactionalTestCase; import io.ebean.Database;
import io.ebean.text.csv.CsvReader; import io.ebean.plugin.BeanType;
import io.ebean.util.IOUtils; import io.ebean.util.IOUtils;
import org.example.Country;
import org.example.Customer;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.tests.model.basic.Customer;
import org.tests.model.basic.ResetBasicData;
import java.io.Reader; import java.io.Reader;
import java.net.URL; import java.net.URL;
import java.util.Locale; import java.util.Locale;
public class TestCsvReader extends TransactionalTestCase { class TestCsvReader {
@Test @Test
public void test() throws Exception { void test() throws Exception {
ResetBasicData.reset(); Country country = new Country();
country.code("NZ");
DB.save(country);
Country au = new Country();
au.code("AU");
DB.save(au);
URL resource = TestCsvReaderWithCallback.class.getResource("/test1.csv"); URL resource = TestCsvReaderWithCallback.class.getResource("/test1.csv");
try (Reader reader = IOUtils.newReader(resource.openStream())){ try (Reader reader = IOUtils.newReader(resource.openStream())) {
CsvReader<Customer> csvReader = DB.getDefault().createCsvReader(Customer.class); Database database = DB.getDefault();
BeanType<Customer> type = database.pluginApi().beanType(Customer.class);
TCsvReader<Customer> csvReader = new TCsvReader<>(database, type);
//CsvReader<Customer> csvReader = DB.getDefault().createCsvReader(Customer.class);
csvReader.setPersistBatchSize(2); csvReader.setPersistBatchSize(2);
@@ -0,0 +1,47 @@
package io.ebeaninternal.server.text.csv;
import org.junit.jupiter.api.Test;
public class TestCsvReaderWithCallback {
@Test
public void test() throws Throwable {
// URL resource = TestCsvReaderWithCallback.class.getResource("/test1.csv");
// try (Reader reader = IOUtils.newReader(resource.openStream())) {
//
// CsvReader<Customer> csvReader = DB.getDefault().createCsvReader(Customer.class);
//
// csvReader.setPersistBatchSize(2);
// csvReader.setLogInfoFrequency(3);
//
// csvReader.addIgnore();
// // csvReader.addProperty("id");
// csvReader.addProperty("status");
// csvReader.addProperty("name");
// csvReader.addDateTime("anniversary", "dd-MMM-yyyy", Locale.ENGLISH);
// csvReader.addProperty("billingAddress.line1");
// csvReader.addProperty("billingAddress.city");
// // processor.addReference("billingAddress.country.code");
// csvReader.addProperty("billingAddress.country.code");
//
// int before = DB.find(Customer.class).findCount();
//
// csvReader.process(reader, new DefaultCsvCallback<Customer>() {
//
// @Override
// public void processBean(int row, String[] lineContent, Customer cust) {
//
// server.save(cust.getBillingAddress(), transaction);
// server.save(cust, transaction);
//
// }
//
// });
//
// int after = DB.find(Customer.class).findCount();
// Assertions.assertThat(after).isEqualTo(before + 9);
// }
}
}
@@ -1,14 +1,12 @@
package org.tests.text.csv; package io.ebeaninternal.server.text.csv;
import io.ebean.xtest.BaseTestCase;
import io.ebean.text.TimeStringParser;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.sql.Time; import java.sql.Time;
import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.Assertions.*;
public class TestTimeStringParser extends BaseTestCase { public class TimeStringParserTest {
@Test @Test
public void testSimple() { public void testSimple() {
@@ -0,0 +1,48 @@
package org.example;
import io.ebean.annotation.WhenCreated;
import javax.persistence.Id;
import javax.persistence.MappedSuperclass;
import javax.persistence.Version;
import java.time.Instant;
@MappedSuperclass
public abstract class BaseEntity {
@Id
long id;
@Version
long version;
@WhenCreated
Instant whenCreated;
public long id() {
return id;
}
public BaseEntity id(long id) {
this.id = id;
return this;
}
public long version() {
return version;
}
public BaseEntity version(long version) {
this.version = version;
return this;
}
public Instant whenCreated() {
return whenCreated;
}
public BaseEntity whenCreated(Instant whenCreated) {
this.whenCreated = whenCreated;
return this;
}
}
@@ -0,0 +1,31 @@
package org.example;
import javax.persistence.Entity;
import javax.persistence.Id;
@Entity
public class Country {
@Id
String code;
String name;
public String code() {
return code;
}
public Country code(String code) {
this.code = code;
return this;
}
public String name() {
return name;
}
public Country name(String name) {
this.name = name;
return this;
}
}
@@ -0,0 +1,63 @@
package org.example;
import javax.persistence.*;
import java.time.LocalDate;
import static javax.persistence.CascadeType.PERSIST;
@Entity
public class Customer extends BaseEntity {
enum Status {
NEW,
ACTIVE,
INACTIVE
}
@Enumerated(EnumType.STRING)
Status status;
String name;
LocalDate anniversary;
@ManyToOne(optional = false, cascade = PERSIST)
DAddress billingAddress;
public Status status() {
return status;
}
public Customer status(Status status) {
this.status = status;
return this;
}
public String name() {
return name;
}
public Customer name(String name) {
this.name = name;
return this;
}
public LocalDate anniversary() {
return anniversary;
}
public Customer anniversary(LocalDate anniversary) {
this.anniversary = anniversary;
return this;
}
public DAddress billingAddress() {
return billingAddress;
}
public Customer billingAddress(DAddress billingAddress) {
this.billingAddress = billingAddress;
return this;
}
}
@@ -0,0 +1,54 @@
package org.example;
import javax.persistence.Entity;
import javax.persistence.ManyToOne;
import static javax.persistence.CascadeType.PERSIST;
@Entity
public class DAddress extends BaseEntity {
String line1;
String line2;
String city;
@ManyToOne(optional = false)
Country country;
public String line1() {
return line1;
}
public DAddress line1(String line1) {
this.line1 = line1;
return this;
}
public String line2() {
return line2;
}
public DAddress line2(String line2) {
this.line2 = line2;
return this;
}
public String city() {
return city;
}
public DAddress city(String city) {
this.city = city;
return this;
}
public Country country() {
return country;
}
public DAddress country(Country country) {
this.country = country;
return this;
}
}
@@ -0,0 +1,4 @@
ebean:
test:
platform: h2
ddlMode: dropCreate
@@ -356,10 +356,6 @@ public class TDSpiEbeanServer extends TDSpiServer implements SpiEbeanServer {
return null; return null;
} }
@Override
public <T> CsvReader<T> createCsvReader(Class<T> beanType) {
return null;
}
@Override @Override
public <T> Query<T> createNamedQuery(Class<T> beanType, String namedQuery) { public <T> Query<T> createNamedQuery(Class<T> beanType, String namedQuery) {
@@ -103,10 +103,6 @@ public class TDSpiServer implements SpiServer {
return null; return null;
} }
@Override
public <T> CsvReader<T> createCsvReader(Class<T> beanType) {
return null;
}
@Override @Override
public <T> UpdateQuery<T> update(Class<T> beanType) { public <T> UpdateQuery<T> update(Class<T> beanType) {
@@ -1,59 +0,0 @@
package org.tests.text.csv;
import io.ebean.DB;
import io.ebean.xtest.base.TransactionalTestCase;
import io.ebean.text.csv.CsvReader;
import io.ebean.text.csv.DefaultCsvCallback;
import io.ebean.util.IOUtils;
import org.junit.jupiter.api.Test;
import org.tests.model.basic.Customer;
import java.io.Reader;
import java.net.URL;
import java.util.Locale;
import static org.assertj.core.api.Assertions.assertThat;
public class TestCsvReaderWithCallback extends TransactionalTestCase {
@Test
public void test() throws Throwable {
URL resource = TestCsvReaderWithCallback.class.getResource("/test1.csv");
try (Reader reader = IOUtils.newReader(resource.openStream())) {
CsvReader<Customer> csvReader = DB.getDefault().createCsvReader(Customer.class);
csvReader.setPersistBatchSize(2);
csvReader.setLogInfoFrequency(3);
csvReader.addIgnore();
// csvReader.addProperty("id");
csvReader.addProperty("status");
csvReader.addProperty("name");
csvReader.addDateTime("anniversary", "dd-MMM-yyyy", Locale.ENGLISH);
csvReader.addProperty("billingAddress.line1");
csvReader.addProperty("billingAddress.city");
// processor.addReference("billingAddress.country.code");
csvReader.addProperty("billingAddress.country.code");
int before = DB.find(Customer.class).findCount();
csvReader.process(reader, new DefaultCsvCallback<Customer>() {
@Override
public void processBean(int row, String[] lineContent, Customer cust) {
server.save(cust.getBillingAddress(), transaction);
server.save(cust, transaction);
}
});
int after = DB.find(Customer.class).findCount();
assertThat(after).isEqualTo(before + 9);
}
}
}
+1
View File
@@ -89,6 +89,7 @@
<module>ebean-redis</module> <module>ebean-redis</module>
<module>platforms</module> <module>platforms</module>
<module>composites</module> <module>composites</module>
<module>ebean-csv-reader</module>
<!-- <module>ebean-kotlin</module>--> <!-- <module>ebean-kotlin</module>-->
</modules> </modules>