perf[storage]: 增加解析csv文件支持

storage/pom.xml

@@ -57,6 +57,7 @@
         <fastjson.version>1.2.51</fastjson.version>
         <!-- office文档解析包 -->
         <poi.version>4.1.2</poi.version>
+        <csv.version>1.9.0</csv.version>
         <!-- 字节码增强 -->
         <javassist.version>3.27.0-GA</javassist.version>
         <bytebuddy.version>1.10.22</bytebuddy.version>
@@ -150,6 +151,11 @@
             <artifactId>poi-ooxml</artifactId>
             <version>${poi.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-csv</artifactId>
+            <version>${csv.version}</version>
+        </dependency>
 
         <!-- slf4j-api -->
         <dependency>

storage/src/main/java/com/zfoo/storage/interpreter/ResourceReader.java

@@ -21,7 +21,8 @@ import com.zfoo.storage.model.anno.Id;
 import com.zfoo.storage.model.resource.ResourceData;
 import com.zfoo.storage.model.resource.ResourceEnum;
 import com.zfoo.storage.strategy.*;
-import com.zfoo.storage.util.ExcelToJsonUtils;
+import com.zfoo.storage.util.CsvReaderUtils;
+import com.zfoo.storage.util.ExcelReaderUtils;
 import org.apache.poi.ss.usermodel.Workbook;
 import org.apache.poi.ss.usermodel.WorkbookFactory;
 import org.springframework.context.support.ConversionServiceFactoryBean;
@@ -63,7 +64,9 @@ public class ResourceReader implements IResourceReader {
         if (resourceEnum == ResourceEnum.JSON) {
             resource = JsonUtils.string2Object(StringUtils.bytesToString(IOUtils.toByteArray(inputStream)), ResourceData.class);
         } else if (resourceEnum == ResourceEnum.EXCEL_XLS || resourceEnum == ResourceEnum.EXCEL_XLSX) {
-            resource = ExcelToJsonUtils.readResourceDataFromExcel(inputStream, clazz.getSimpleName());
+            resource = ExcelReaderUtils.readResourceDataFromExcel(inputStream, clazz.getSimpleName());
+        } else if (resourceEnum == ResourceEnum.CSV) {
+            resource = CsvReaderUtils.readResourceDataFromCSV(inputStream, clazz.getSimpleName());
         } else {
             throw new RunException("不支持文件[{}]的配置类型[{}]", clazz.getSimpleName(), suffix);
         }
@@ -76,7 +79,7 @@ public class ResourceReader implements IResourceReader {
         var iterator = resource.getRows().iterator();
         // 从ROW_SERVER这行开始读取数据
         while (iterator.hasNext()) {
-            var columns = iterator.next().getColumns();
+            var columns = iterator.next();
             var instance = ReflectionUtils.newInstance(clazz);
 
             for (var fieldInfo : fieldInfos) {

storage/src/main/java/com/zfoo/storage/model/resource/ResourceData.java

@@ -27,9 +27,9 @@ public class ResourceData {
     // 配置表字段名
     private List<ResourceHeader> headers = new ArrayList<>();
     // 配置表数据
-    private List<ResourceRow> rows = new ArrayList<>();
+    private List<List<String>> rows = new ArrayList<>();
 
-    public static ResourceData valueOf(String name, List<ResourceHeader> headers, List<ResourceRow> rows) {
+    public static ResourceData valueOf(String name, List<ResourceHeader> headers, List<List<String>> rows) {
         var resourceData = new ResourceData();
         resourceData.name = name;
         resourceData.headers = headers;
@@ -53,11 +53,11 @@ public class ResourceData {
         this.headers = headers;
     }
 
-    public List<ResourceRow> getRows() {
+    public List<List<String>> getRows() {
         return rows;
     }
 
-    public void setRows(List<ResourceRow> rows) {
+    public void setRows(List<List<String>> rows) {
         this.rows = rows;
     }
 

storage/src/main/java/com/zfoo/storage/model/resource/ResourceEnum.java

@@ -33,6 +33,8 @@ public enum ResourceEnum {
 
     JSON("json"),
 
+    CSV("csv"),
+
     ;
 
     private static Map<String, ResourceEnum> typeMap = new HashMap<>();

storage/src/main/java/com/zfoo/storage/model/resource/ResourceRow.java

@@ -1,51 +0,0 @@
-/*
- * Copyright (C) 2020 The zfoo Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and limitations under the License.
- */
-package com.zfoo.storage.model.resource;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * @author godotg
- * @version 4.0
- */
-public class ResourceRow {
-
-    private int row;
-
-    private List<String> columns = new ArrayList<>();
-
-    public static ResourceRow valueOf(int row, List<String> columns) {
-        var resourceRow = new ResourceRow();
-        resourceRow.row = row;
-        resourceRow.columns = columns;
-        return resourceRow;
-    }
-
-    public int getRow() {
-        return row;
-    }
-
-    public void setRow(int row) {
-        this.row = row;
-    }
-
-    public List<String> getColumns() {
-        return columns;
-    }
-
-    public void setColumns(List<String> columns) {
-        this.columns = columns;
-    }
-
-}

storage/src/main/java/com/zfoo/storage/util/CsvReaderUtils.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2020 The zfoo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and limitations under the License.
+ */
+
+package com.zfoo.storage.util;
+
+import com.zfoo.protocol.exception.RunException;
+import com.zfoo.protocol.util.StringUtils;
+import com.zfoo.storage.model.resource.ResourceData;
+import com.zfoo.storage.model.resource.ResourceHeader;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.*;
+
+/**
+ * @author meiwei666
+ * @version 4.0
+ */
+public class CsvReaderUtils {
+
+    /**
+     * 构建配置表消息头
+     * @param iterator
+     * @param fileName
+     * @return
+     */
+    private static List<ResourceHeader> getHeaders(Iterator<CSVRecord> iterator, String fileName) {
+        // 获取配置表的有效列名称，默认第一行就是字段名称
+        var fieldRow = iterator.next();
+        if (fieldRow == null) {
+            throw new RunException("无法获取资源[class:{}]的Excel文件的属性控制列", fileName);
+        }
+        //默认第二行字段类型
+        var typeRow = iterator.next();
+        if (typeRow == null) {
+            throw new RunException("无法获取资源[class:{}]的Excel文件的类型控制列", fileName);
+        }
+        var descRow = iterator.next();
+
+        List<ResourceHeader> headers = new ArrayList<>();
+        for (int i = 0; i < fieldRow.size(); i++) {
+            String fieldName = fieldRow.get(i);
+            if (fieldName == null) {
+                throw new RunException("{}列名不能为空，第{}列没有配置名字", fileName ,i + 1);
+            }
+            String filedType = typeRow.get(i);
+            if (filedType == null) {
+                throw new RunException("{}列类型不能为空，第{}列没有配置类型", fileName ,i + 1);
+            }
+//            String desc = descRecord.get(i);
+            headers.add(ResourceHeader.valueOf(fieldName, filedType, i ));
+        }
+        return headers;
+    }
+
+    public static ResourceData readResourceDataFromCSV(InputStream input, String fileName){
+        CSVParser records = parseCsv(input, fileName);
+        Iterator<CSVRecord> iter = records.iterator();
+        List<ResourceHeader> headers = getHeaders(iter, fileName);
+        List<List<String>> rows = new ArrayList<>();
+        while (iter.hasNext()) {
+            CSVRecord record = iter.next();
+            List<String> data = new ArrayList<>();
+            for (ResourceHeader header : headers) {
+                String value = record.get(header.getIndex());
+                if (StringUtils.isBlank(value)) {
+                    value = StringUtils.EMPTY;
+                }
+                data.add(value);
+            }
+            rows.add(data);
+        }
+        return ResourceData.valueOf(fileName, headers, rows);
+    }
+
+    private static CSVParser parseCsv(InputStream input, String fileName) {
+        try {
+            return CSVFormat.EXCEL.parse(new InputStreamReader(input));
+        } catch (IOException e) {
+            throw new RunException("静态资源[{}]异常，无法读取文件", fileName);
+        }
+    }
+}

storage/src/main/java/com/zfoo/storage/util/ExcelReaderUtils.java

@@ -20,25 +20,19 @@ import com.zfoo.protocol.util.StringUtils;
 import com.zfoo.storage.model.resource.ResourceData;
 import com.zfoo.storage.model.resource.ResourceEnum;
 import com.zfoo.storage.model.resource.ResourceHeader;
-import com.zfoo.storage.model.resource.ResourceRow;
-import org.apache.poi.ss.usermodel.Sheet;
-import org.apache.poi.ss.usermodel.Workbook;
-import org.apache.poi.ss.usermodel.WorkbookFactory;
+import org.apache.poi.ss.usermodel.*;
 
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Objects;
+import java.util.*;
 import java.util.stream.Collectors;
 
 /**
  * @author meiwei666
  * @version 4.0
  */
-public class ExcelToJsonUtils {
+public class ExcelReaderUtils {
 
     public static void excelConvertJson(String inputDir, String outputDir) throws IOException {
         var listFiles = FileUtils.getAllReadableFiles(new File(inputDir))
@@ -54,7 +48,7 @@ public class ExcelToJsonUtils {
 
             var outputFilePath = FileUtils.joinPath(outputDir, jsonFileName);
             FileUtils.deleteFile(new File(outputFilePath));
-            FileUtils.writeStringToFile(new File(outputFilePath), JsonUtils.object2StringPrettyPrinter(resourceData), true);
+            FileUtils.writeStringToFile(new File(outputFilePath), JsonUtils.object2String(resourceData), true);
         }
     }
 
@@ -63,16 +57,17 @@ public class ExcelToJsonUtils {
         var wb = createWorkbook(inputStream, fileName);
         // 默认取到第一个sheet页
         var sheet = wb.getSheetAt(0);
+        var iterator = sheet.iterator();
         //设置所有列
-        var headers = getHeaders(sheet, fileName);
+        var headers = getHeaders(iterator, fileName);
 
         // 行数定位到有效数据行，默认是第四行为有效数据行
-        var iterator = sheet.iterator();
-        iterator.next();
-        iterator.next();
-        iterator.next();
+//        var iterator = sheet.iterator();
+//        iterator.next();
+//        iterator.next();
+//        iterator.next();
         // 从ROW_SERVER这行开始读取数据
-        List<ResourceRow> rows = new ArrayList<>();
+        List<List<String>> rows = new ArrayList<>();
         while (iterator.hasNext()) {
             var row = iterator.next();
 
@@ -87,14 +82,14 @@ public class ExcelToJsonUtils {
                 var content = CellUtils.getCellStringValue(cell);
                 columns.add(content);
             }
-            rows.add(ResourceRow.valueOf(row.getRowNum() + 1, columns));
+            rows.add(columns);
         }
         return ResourceData.valueOf(fileName, headers, rows);
     }
 
     // 只读取代码里写的字段
-    private static List<ResourceHeader> getHeaders(Sheet sheet, String fileName) {
-        var iterator = sheet.iterator();
+    private static List<ResourceHeader> getHeaders(Iterator<Row> iterator, String fileName) {
+//        var iterator = sheet.iterator();
         // 获取配置表的有效列名称，默认第一行就是字段名称
         var fieldRow = iterator.next();
         if (fieldRow == null) {
@@ -105,7 +100,7 @@ public class ExcelToJsonUtils {
         if (typeRow == null) {
             throw new RunException("无法获取资源[class:{}]的Excel文件的类型控制列", fileName);
         }
-
+        var desRow = iterator.next();
         var headerList = new ArrayList<ResourceHeader>();
         var cellFieldMap = new HashMap<String, Integer>();
         for (var i = 0; i < fieldRow.getLastCellNum(); i++) {

storage/src/test/java/com/zfoo/storage/ApplicationTest.java

@@ -47,6 +47,7 @@ public class ApplicationTest {
 
         var studentManager = context.getBean(StudentManager.class);
         var studentResources = studentManager.studentResources;
+        var studentCsvResources = studentManager.studentCsvResources;
         // 类名称和Excel名称必须完全一致，Excel的列名称必须对应对象的属性名称
         // 类名称和Excel名称必须完全一致，Excel的列名称必须对应对象的属性名称
         for (StudentResource resource : studentResources.getAll()) {
@@ -64,6 +65,10 @@ public class ApplicationTest {
         var valuesByIndex = studentResources.getIndex("name", "james0");
         logger.info(JsonUtils.object2String(valuesByIndex));
 
+        // 通过索引找对应的行
+        var csvValuesByIndex = studentCsvResources.getIndex("name", "james0");
+        logger.info(JsonUtils.object2String(csvValuesByIndex));
+
         // Excel的映射内容需要在被Spring管理的bean的方法上加上@ResInjection注解，即可自动注入Excel对应的对象
         var testManager = context.getBean(TestManager.class);
         var testResources = testManager.testResources;

storage/src/test/java/com/zfoo/storage/StudentManager.java

@@ -15,6 +15,7 @@ package com.zfoo.storage;
 
 import com.zfoo.storage.model.anno.ResInjection;
 import com.zfoo.storage.model.vo.Storage;
+import com.zfoo.storage.resource.StudentCsvResource;
 import com.zfoo.storage.resource.StudentResource;
 import org.springframework.stereotype.Component;
 
@@ -27,5 +28,7 @@ public class StudentManager {
 
     @ResInjection
     public Storage<Integer, StudentResource> studentResources;
+    @ResInjection
+    public Storage<Integer, StudentCsvResource> studentCsvResources;
 
 }

storage/src/test/java/com/zfoo/storage/excel/ExcelTest.java

@@ -13,7 +13,7 @@
 
 package com.zfoo.storage.excel;
 
-import com.zfoo.storage.util.ExcelToJsonUtils;
+import com.zfoo.storage.util.ExcelReaderUtils;
 import org.apache.poi.ss.usermodel.Row;
 import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.ss.usermodel.Workbook;
@@ -37,7 +37,7 @@ public class ExcelTest {
     public void excelConvertJson() throws Exception{
         String inputDir = "E:\\workspace\\zfoo\\storage\\src\\test\\resources\\excel";
         String outputDir = "E:\\workspace\\zfoo\\storage\\src\\test\\resources\\excel";
-        ExcelToJsonUtils.excelConvertJson(inputDir, outputDir);
+        ExcelReaderUtils.excelConvertJson(inputDir, outputDir);
     }
 
     @Test

storage/src/test/java/com/zfoo/storage/resource/StudentCsvResource.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2020 The zfoo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and limitations under the License.
+ */
+
+package com.zfoo.storage.resource;
+
+import com.zfoo.storage.model.anno.Id;
+import com.zfoo.storage.model.anno.Index;
+import com.zfoo.storage.model.anno.Resource;
+
+/**
+ * @author godotg
+ * @version 4.0
+ */
+@Resource
+public class StudentCsvResource {
+
+    @Id
+    private int id;
+
+    /**
+     * 索引，默认为可重复的索引
+     */
+    @Index
+    private String name;
+
+    private int age;
+    private float score;
+    private String[] courses;
+    private User[] users;
+    private User user;
+
+    /**
+     * 不想映射的字段必须加上transient关键字，这样就不会从Excel中去找对应的列
+     */
+    private transient String notMapContent;
+
+    public int getId() {
+        return id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public int getAge() {
+        return age;
+    }
+
+    public float getScore() {
+        return score;
+    }
+
+    public String[] getCourses() {
+        return courses;
+    }
+
+    public User[] getUsers() {
+        return users;
+    }
+
+    public User getUser() {
+        return user;
+    }
+
+    public String getNotMapContent() {
+        return notMapContent;
+    }
+}

storage/src/test/resources/excel/StudentCsvResource.csv

@@ -0,0 +1,7 @@
+id,name,age,score,courses,users,user
+string,string,int,float,array,array,string
+des,des,des,des,des,des,des
+1000,james0,10,60.8,"[""History"",""Chiness""]","[{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22},{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22}]","{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22}"
+1001,james1,10,70.1,"[""History"",""Chiness""]","[{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22},{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":23}]","{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22}"
+1002,james2,10,80.2,"[""History"",""Chiness""]","[{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22},{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":24}]","{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22}"
+1003,james3,10,59,"[""History"",""Chiness""]","[{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22},{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":25}]","{""id"":""1000"",""name"":""SunInsanity"",""sex"":""boy"",""age"":22}"