diff --git a/storage/pom.xml b/storage/pom.xml
index 9b0dad2b..bd710ade 100644
--- a/storage/pom.xml
+++ b/storage/pom.xml
@@ -57,6 +57,7 @@
1.2.51
4.1.2
+ 1.9.0
3.27.0-GA
1.10.22
@@ -150,6 +151,11 @@
poi-ooxml
${poi.version}
+
+ org.apache.commons
+ commons-csv
+ ${csv.version}
+
diff --git a/storage/src/main/java/com/zfoo/storage/interpreter/ResourceReader.java b/storage/src/main/java/com/zfoo/storage/interpreter/ResourceReader.java
index ec40432a..d8627393 100644
--- a/storage/src/main/java/com/zfoo/storage/interpreter/ResourceReader.java
+++ b/storage/src/main/java/com/zfoo/storage/interpreter/ResourceReader.java
@@ -21,7 +21,8 @@ import com.zfoo.storage.model.anno.Id;
import com.zfoo.storage.model.resource.ResourceData;
import com.zfoo.storage.model.resource.ResourceEnum;
import com.zfoo.storage.strategy.*;
-import com.zfoo.storage.util.ExcelToJsonUtils;
+import com.zfoo.storage.util.CsvReaderUtils;
+import com.zfoo.storage.util.ExcelReaderUtils;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.springframework.context.support.ConversionServiceFactoryBean;
@@ -63,7 +64,9 @@ public class ResourceReader implements IResourceReader {
if (resourceEnum == ResourceEnum.JSON) {
resource = JsonUtils.string2Object(StringUtils.bytesToString(IOUtils.toByteArray(inputStream)), ResourceData.class);
} else if (resourceEnum == ResourceEnum.EXCEL_XLS || resourceEnum == ResourceEnum.EXCEL_XLSX) {
- resource = ExcelToJsonUtils.readResourceDataFromExcel(inputStream, clazz.getSimpleName());
+ resource = ExcelReaderUtils.readResourceDataFromExcel(inputStream, clazz.getSimpleName());
+ } else if (resourceEnum == ResourceEnum.CSV) {
+ resource = CsvReaderUtils.readResourceDataFromCSV(inputStream, clazz.getSimpleName());
} else {
throw new RunException("不支持文件[{}]的配置类型[{}]", clazz.getSimpleName(), suffix);
}
@@ -76,7 +79,7 @@ public class ResourceReader implements IResourceReader {
var iterator = resource.getRows().iterator();
// 从ROW_SERVER这行开始读取数据
while (iterator.hasNext()) {
- var columns = iterator.next().getColumns();
+ var columns = iterator.next();
var instance = ReflectionUtils.newInstance(clazz);
for (var fieldInfo : fieldInfos) {
diff --git a/storage/src/main/java/com/zfoo/storage/model/resource/ResourceData.java b/storage/src/main/java/com/zfoo/storage/model/resource/ResourceData.java
index 3b1fdb11..23d683bc 100644
--- a/storage/src/main/java/com/zfoo/storage/model/resource/ResourceData.java
+++ b/storage/src/main/java/com/zfoo/storage/model/resource/ResourceData.java
@@ -27,9 +27,9 @@ public class ResourceData {
// 配置表字段名
private List headers = new ArrayList<>();
// 配置表数据
- private List rows = new ArrayList<>();
+ private List> rows = new ArrayList<>();
- public static ResourceData valueOf(String name, List headers, List rows) {
+ public static ResourceData valueOf(String name, List headers, List> rows) {
var resourceData = new ResourceData();
resourceData.name = name;
resourceData.headers = headers;
@@ -53,11 +53,11 @@ public class ResourceData {
this.headers = headers;
}
- public List getRows() {
+ public List> getRows() {
return rows;
}
- public void setRows(List rows) {
+ public void setRows(List> rows) {
this.rows = rows;
}
diff --git a/storage/src/main/java/com/zfoo/storage/model/resource/ResourceEnum.java b/storage/src/main/java/com/zfoo/storage/model/resource/ResourceEnum.java
index 53e34e44..e203b875 100644
--- a/storage/src/main/java/com/zfoo/storage/model/resource/ResourceEnum.java
+++ b/storage/src/main/java/com/zfoo/storage/model/resource/ResourceEnum.java
@@ -33,6 +33,8 @@ public enum ResourceEnum {
JSON("json"),
+ CSV("csv"),
+
;
private static Map typeMap = new HashMap<>();
diff --git a/storage/src/main/java/com/zfoo/storage/model/resource/ResourceRow.java b/storage/src/main/java/com/zfoo/storage/model/resource/ResourceRow.java
deleted file mode 100644
index 764661dc..00000000
--- a/storage/src/main/java/com/zfoo/storage/model/resource/ResourceRow.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright (C) 2020 The zfoo Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and limitations under the License.
- */
-package com.zfoo.storage.model.resource;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * @author godotg
- * @version 4.0
- */
-public class ResourceRow {
-
- private int row;
-
- private List columns = new ArrayList<>();
-
- public static ResourceRow valueOf(int row, List columns) {
- var resourceRow = new ResourceRow();
- resourceRow.row = row;
- resourceRow.columns = columns;
- return resourceRow;
- }
-
- public int getRow() {
- return row;
- }
-
- public void setRow(int row) {
- this.row = row;
- }
-
- public List getColumns() {
- return columns;
- }
-
- public void setColumns(List columns) {
- this.columns = columns;
- }
-
-}
diff --git a/storage/src/main/java/com/zfoo/storage/util/CsvReaderUtils.java b/storage/src/main/java/com/zfoo/storage/util/CsvReaderUtils.java
new file mode 100644
index 00000000..f27dde9f
--- /dev/null
+++ b/storage/src/main/java/com/zfoo/storage/util/CsvReaderUtils.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2020 The zfoo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and limitations under the License.
+ */
+
+package com.zfoo.storage.util;
+
+import com.zfoo.protocol.exception.RunException;
+import com.zfoo.protocol.util.StringUtils;
+import com.zfoo.storage.model.resource.ResourceData;
+import com.zfoo.storage.model.resource.ResourceHeader;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.*;
+
+/**
+ * @author meiwei666
+ * @version 4.0
+ */
+public class CsvReaderUtils {
+
+ /**
+ * 构建配置表消息头
+ * @param iterator
+ * @param fileName
+ * @return
+ */
+ private static List getHeaders(Iterator iterator, String fileName) {
+ // 获取配置表的有效列名称，默认第一行就是字段名称
+ var fieldRow = iterator.next();
+ if (fieldRow == null) {
+ throw new RunException("无法获取资源[class:{}]的Excel文件的属性控制列", fileName);
+ }
+ //默认第二行字段类型
+ var typeRow = iterator.next();
+ if (typeRow == null) {
+ throw new RunException("无法获取资源[class:{}]的Excel文件的类型控制列", fileName);
+ }
+ var descRow = iterator.next();
+
+ List headers = new ArrayList<>();
+ for (int i = 0; i < fieldRow.size(); i++) {
+ String fieldName = fieldRow.get(i);
+ if (fieldName == null) {
+ throw new RunException("{}列名不能为空，第{}列没有配置名字", fileName ,i + 1);
+ }
+ String filedType = typeRow.get(i);
+ if (filedType == null) {
+ throw new RunException("{}列类型不能为空，第{}列没有配置类型", fileName ,i + 1);
+ }
+// String desc = descRecord.get(i);
+ headers.add(ResourceHeader.valueOf(fieldName, filedType, i ));
+ }
+ return headers;
+ }
+
+ public static ResourceData readResourceDataFromCSV(InputStream input, String fileName){
+ CSVParser records = parseCsv(input, fileName);
+ Iterator iter = records.iterator();
+ List headers = getHeaders(iter, fileName);
+ List> rows = new ArrayList<>();
+ while (iter.hasNext()) {
+ CSVRecord record = iter.next();
+ List data = new ArrayList<>();
+ for (ResourceHeader header : headers) {
+ String value = record.get(header.getIndex());
+ if (StringUtils.isBlank(value)) {
+ value = StringUtils.EMPTY;
+ }
+ data.add(value);
+ }
+ rows.add(data);
+ }
+ return ResourceData.valueOf(fileName, headers, rows);
+ }
+
+ private static CSVParser parseCsv(InputStream input, String fileName) {
+ try {
+ return CSVFormat.EXCEL.parse(new InputStreamReader(input));
+ } catch (IOException e) {
+ throw new RunException("静态资源[{}]异常，无法读取文件", fileName);
+ }
+ }
+}
diff --git a/storage/src/main/java/com/zfoo/storage/util/ExcelToJsonUtils.java b/storage/src/main/java/com/zfoo/storage/util/ExcelReaderUtils.java
similarity index 86%
rename from storage/src/main/java/com/zfoo/storage/util/ExcelToJsonUtils.java
rename to storage/src/main/java/com/zfoo/storage/util/ExcelReaderUtils.java
index b5bcad67..41f70769 100644
--- a/storage/src/main/java/com/zfoo/storage/util/ExcelToJsonUtils.java
+++ b/storage/src/main/java/com/zfoo/storage/util/ExcelReaderUtils.java
@@ -20,25 +20,19 @@ import com.zfoo.protocol.util.StringUtils;
import com.zfoo.storage.model.resource.ResourceData;
import com.zfoo.storage.model.resource.ResourceEnum;
import com.zfoo.storage.model.resource.ResourceHeader;
-import com.zfoo.storage.model.resource.ResourceRow;
-import org.apache.poi.ss.usermodel.Sheet;
-import org.apache.poi.ss.usermodel.Workbook;
-import org.apache.poi.ss.usermodel.WorkbookFactory;
+import org.apache.poi.ss.usermodel.*;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Objects;
+import java.util.*;
import java.util.stream.Collectors;
/**
* @author meiwei666
* @version 4.0
*/
-public class ExcelToJsonUtils {
+public class ExcelReaderUtils {
public static void excelConvertJson(String inputDir, String outputDir) throws IOException {
var listFiles = FileUtils.getAllReadableFiles(new File(inputDir))
@@ -54,7 +48,7 @@ public class ExcelToJsonUtils {
var outputFilePath = FileUtils.joinPath(outputDir, jsonFileName);
FileUtils.deleteFile(new File(outputFilePath));
- FileUtils.writeStringToFile(new File(outputFilePath), JsonUtils.object2StringPrettyPrinter(resourceData), true);
+ FileUtils.writeStringToFile(new File(outputFilePath), JsonUtils.object2String(resourceData), true);
}
}
@@ -63,16 +57,17 @@ public class ExcelToJsonUtils {
var wb = createWorkbook(inputStream, fileName);
// 默认取到第一个sheet页
var sheet = wb.getSheetAt(0);
+ var iterator = sheet.iterator();
//设置所有列
- var headers = getHeaders(sheet, fileName);
+ var headers = getHeaders(iterator, fileName);
// 行数定位到有效数据行，默认是第四行为有效数据行
- var iterator = sheet.iterator();
- iterator.next();
- iterator.next();
- iterator.next();
+// var iterator = sheet.iterator();
+// iterator.next();
+// iterator.next();
+// iterator.next();
// 从ROW_SERVER这行开始读取数据
- List rows = new ArrayList<>();
+ List> rows = new ArrayList<>();
while (iterator.hasNext()) {
var row = iterator.next();
@@ -87,14 +82,14 @@ public class ExcelToJsonUtils {
var content = CellUtils.getCellStringValue(cell);
columns.add(content);
}
- rows.add(ResourceRow.valueOf(row.getRowNum() + 1, columns));
+ rows.add(columns);
}
return ResourceData.valueOf(fileName, headers, rows);
}
// 只读取代码里写的字段
- private static List getHeaders(Sheet sheet, String fileName) {
- var iterator = sheet.iterator();
+ private static List getHeaders(Iterator iterator, String fileName) {
+// var iterator = sheet.iterator();
// 获取配置表的有效列名称，默认第一行就是字段名称
var fieldRow = iterator.next();
if (fieldRow == null) {
@@ -105,7 +100,7 @@ public class ExcelToJsonUtils {
if (typeRow == null) {
throw new RunException("无法获取资源[class:{}]的Excel文件的类型控制列", fileName);
}
-
+ var desRow = iterator.next();
var headerList = new ArrayList();
var cellFieldMap = new HashMap();
for (var i = 0; i < fieldRow.getLastCellNum(); i++) {
diff --git a/storage/src/test/java/com/zfoo/storage/ApplicationTest.java b/storage/src/test/java/com/zfoo/storage/ApplicationTest.java
index 4250cdcf..c7ac921b 100644
--- a/storage/src/test/java/com/zfoo/storage/ApplicationTest.java
+++ b/storage/src/test/java/com/zfoo/storage/ApplicationTest.java
@@ -47,6 +47,7 @@ public class ApplicationTest {
var studentManager = context.getBean(StudentManager.class);
var studentResources = studentManager.studentResources;
+ var studentCsvResources = studentManager.studentCsvResources;
// 类名称和Excel名称必须完全一致，Excel的列名称必须对应对象的属性名称
// 类名称和Excel名称必须完全一致，Excel的列名称必须对应对象的属性名称
for (StudentResource resource : studentResources.getAll()) {
@@ -64,6 +65,10 @@ public class ApplicationTest {
var valuesByIndex = studentResources.getIndex("name", "james0");
logger.info(JsonUtils.object2String(valuesByIndex));
+ // 通过索引找对应的行
+ var csvValuesByIndex = studentCsvResources.getIndex("name", "james0");
+ logger.info(JsonUtils.object2String(csvValuesByIndex));
+
// Excel的映射内容需要在被Spring管理的bean的方法上加上@ResInjection注解，即可自动注入Excel对应的对象
var testManager = context.getBean(TestManager.class);
var testResources = testManager.testResources;
diff --git a/storage/src/test/java/com/zfoo/storage/StudentManager.java b/storage/src/test/java/com/zfoo/storage/StudentManager.java
index 06113f8b..ef07e04d 100644
--- a/storage/src/test/java/com/zfoo/storage/StudentManager.java
+++ b/storage/src/test/java/com/zfoo/storage/StudentManager.java
@@ -15,6 +15,7 @@ package com.zfoo.storage;
import com.zfoo.storage.model.anno.ResInjection;
import com.zfoo.storage.model.vo.Storage;
+import com.zfoo.storage.resource.StudentCsvResource;
import com.zfoo.storage.resource.StudentResource;
import org.springframework.stereotype.Component;
@@ -27,5 +28,7 @@ public class StudentManager {
@ResInjection
public Storage studentResources;
+ @ResInjection
+ public Storage studentCsvResources;
}
diff --git a/storage/src/test/java/com/zfoo/storage/excel/ExcelTest.java b/storage/src/test/java/com/zfoo/storage/excel/ExcelTest.java
index 46d84771..26780eba 100644
--- a/storage/src/test/java/com/zfoo/storage/excel/ExcelTest.java
+++ b/storage/src/test/java/com/zfoo/storage/excel/ExcelTest.java
@@ -13,7 +13,7 @@
package com.zfoo.storage.excel;
-import com.zfoo.storage.util.ExcelToJsonUtils;
+import com.zfoo.storage.util.ExcelReaderUtils;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
@@ -37,7 +37,7 @@ public class ExcelTest {
public void excelConvertJson() throws Exception{
String inputDir = "E:\\workspace\\zfoo\\storage\\src\\test\\resources\\excel";
String outputDir = "E:\\workspace\\zfoo\\storage\\src\\test\\resources\\excel";
- ExcelToJsonUtils.excelConvertJson(inputDir, outputDir);
+ ExcelReaderUtils.excelConvertJson(inputDir, outputDir);
}
@Test
diff --git a/storage/src/test/java/com/zfoo/storage/resource/StudentCsvResource.java b/storage/src/test/java/com/zfoo/storage/resource/StudentCsvResource.java
new file mode 100644
index 00000000..eac5d698
--- /dev/null
+++ b/storage/src/test/java/com/zfoo/storage/resource/StudentCsvResource.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2020 The zfoo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and limitations under the License.
+ */
+
+package com.zfoo.storage.resource;
+
+import com.zfoo.storage.model.anno.Id;
+import com.zfoo.storage.model.anno.Index;
+import com.zfoo.storage.model.anno.Resource;
+
+/**
+ * @author godotg
+ * @version 4.0
+ */
+@Resource
+public class StudentCsvResource {
+
+ @Id
+ private int id;
+
+ /**
+ * 索引，默认为可重复的索引
+ */
+ @Index
+ private String name;
+
+ private int age;
+ private float score;
+ private String[] courses;
+ private User[] users;
+ private User user;
+
+ /**
+ * 不想映射的字段必须加上transient关键字，这样就不会从Excel中去找对应的列
+ */
+ private transient String notMapContent;
+
+ public int getId() {
+ return id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public int getAge() {
+ return age;
+ }
+
+ public float getScore() {
+ return score;
+ }
+
+ public String[] getCourses() {
+ return courses;
+ }
+
+ public User[] getUsers() {
+ return users;
+ }
+
+ public User getUser() {
+ return user;
+ }
+
+ public String getNotMapContent() {
+ return notMapContent;
+ }
+}
diff --git a/storage/src/test/resources/StudentResource.xlsx b/storage/src/test/resources/StudentResource.xlsx
deleted file mode 100644
index d7378b7a..00000000
Binary files a/storage/src/test/resources/StudentResource.xlsx and /dev/null differ
diff --git a/storage/src/test/resources/excel/StudentCsvResource.csv b/storage/src/test/resources/excel/StudentCsvResource.csv
new file mode 100644
index 00000000..116b2e8e
--- /dev/null
+++ b/storage/src/test/resources/excel/StudentCsvResource.csv
@@ -0,0 +1,7 @@
+id,name,age,score,courses,users,user
+string,string,int,float,array,array,string
+des,des,des,des,des,des,des
+1000,james0,10,60.8,"[""History"",""Chiness""]","[{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22},{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22}]","{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22}"
+1001,james1,10,70.1,"[""History"",""Chiness""]","[{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22},{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":23}]","{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22}"
+1002,james2,10,80.2,"[""History"",""Chiness""]","[{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22},{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":24}]","{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22}"
+1003,james3,10,59,"[""History"",""Chiness""]","[{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22},{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":25}]","{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22}"