Parse Excel as String only

This commit is contained in:
Andreas Isler
2025-11-18 20:36:53 +01:00
parent 861633d373
commit caa1e4b8f8
9 changed files with 105 additions and 116 deletions
+1 -1
View File
@@ -74,7 +74,7 @@
</profile-state>
</entry>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_11" project-jdk-name="homebrew-17" project-jdk-type="JavaSDK">
<component name="ProjectRootManager" version="2" languageLevel="JDK_17" default="true" project-jdk-name="homebrew-17" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>
@@ -6,10 +6,11 @@ import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.apache.poi.ss.usermodel.DateUtil;
import org.springframework.stereotype.Component;
import java.time.*;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.*;
@@ -28,7 +29,7 @@ public class FlowableExcelMapper {
// excelPath defines the location of the Excel file
// FlowableExcelParser is used to create an ArrayList from the Excel file
// The first row of every sheet contains the path of the variables to be created in the JsonNode
// The 1st row of every sheet contains the path of the variables to be created in the JsonNode
// Delimiter is "." for an ObjectNode and "*" for an ArrayNode
// The 2nd row contains tht type of the variable. Supported types are:
// - String
@@ -39,22 +40,22 @@ public class FlowableExcelMapper {
// All sheets are added to an ArrayNode
// All rows are added to an ArrayNode in the sheets ArrayNode
// All cell values are added to an ObjectNode in the rows ArrayNode
public ArrayNode excelBookToJsonNode(String excelPath) {
ArrayList<ArrayList<ArrayList<Object>>> book = flowableExcelParser.parseExcelBook(excelPath);
public ArrayNode excelBookResourceToJsonNode(String resourcePath) {
ArrayList<ArrayList<ArrayList<String>>> book = flowableExcelParser.parseExcelBookFromResource(resourcePath);
return bookToJsonNode(book);
}
protected ArrayNode bookToJsonNode(ArrayList<ArrayList<ArrayList<Object>>> book) {
protected ArrayNode bookToJsonNode(ArrayList<ArrayList<ArrayList<String>>> book) {
ArrayNode bookNode = objectMapper.createArrayNode();
for (ArrayList<ArrayList<Object>> sheet : book) {
for (ArrayList<ArrayList<String>> sheet : book) {
bookNode.add(sheetToJsonNode(sheet));
}
return bookNode;
}
protected ArrayNode sheetToJsonNode(ArrayList<ArrayList<Object>> rows) {
ArrayList<Object> paths = new ArrayList<>();
ArrayList<Object> types = new ArrayList<>();
protected ArrayNode sheetToJsonNode(ArrayList<ArrayList<String>> rows) {
ArrayList<String> paths = new ArrayList<>();
ArrayList<String> types = new ArrayList<>();
ArrayNode arrayNode = objectMapper.createArrayNode();
for (int rowNum = 0; rowNum < rows.size(); rowNum++) {
if (rowNum == 0) {
@@ -71,25 +72,37 @@ public class FlowableExcelMapper {
return arrayNode;
}
protected JsonNode rowToJsonNode(ArrayList<Object> types, ArrayList<Object> paths, ArrayList<Object> row, JsonNode jsonNode) {
protected JsonNode rowToJsonNode(ArrayList<String> types, ArrayList<String> paths, ArrayList<String> row, JsonNode jsonNode) {
for (int cellNum = 0; cellNum < row.size(); cellNum++) {
Object content = row.get(cellNum);
if (content != null) {
String type = types.get(cellNum).toString();
String cellValue = row.get(cellNum);
if (cellValue == null || cellValue.isEmpty()) {
jsonNode = addNode(jsonNode, paths.get(cellNum), null);
} else {
String type = types.get(cellNum);
Object content;
switch (type) {
case "String": content = content.toString(); break;
case "Boolean": content = Boolean.parseBoolean(content.toString()); break;
case "Integer": content = (int) Double.parseDouble(content.toString()); break;
case "Double": content = Double.parseDouble(content.toString()); break;
case "String": content = cellValue; break;
case "Boolean": content = Boolean.parseBoolean(cellValue); break;
case "Integer": content = Integer.parseInt(cellValue); break;
case "Double": content = Double.parseDouble(cellValue); break;
case "Long": content = Long.parseLong(cellValue); break;
case "Date":
// Excel considers dates as local dates, but we want them as UTC
Date dateTime = DateUtil.getJavaDate(Double.parseDouble(content.toString()));
content = dateTime.toInstant().plusSeconds(ZonedDateTime.now().getOffset().getTotalSeconds());
Instant instant;
try {
LocalDateTime localDateTime = LocalDateTime.parse(cellValue, DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
instant = localDateTime.atZone(ZoneId.systemDefault()).toInstant();
} catch (DateTimeParseException e) {
LocalDate localDate = LocalDate.parse(cellValue, DateTimeFormatter.ofPattern("yyyy-MM-dd"));
instant = localDate.atStartOfDay(ZoneId.systemDefault()).toInstant();
}
content = instant.plusSeconds(ZonedDateTime.now().getOffset().getTotalSeconds());
break;
default: break;
default:
throw new RuntimeException("Variable type not supported: " + type);
}
jsonNode = addNode(jsonNode, paths.get(cellNum), content);
}
jsonNode = addNode(jsonNode, paths.get(cellNum).toString(), content);
}
return jsonNode;
}
@@ -7,6 +7,7 @@ import org.springframework.stereotype.Component;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.LinkedHashMap;
@@ -16,15 +17,15 @@ public class FlowableExcelParser {
// excelPath defines the location of the Excel file
// Return all cells in all sheets with the following restrictions:
// - Max columns per sheet is defined in the first row
// - Max column count per sheet is defined in the first row
// - Max column count starts with the 1st cell and ends with the 1st empty cell
// - Parsing the sheet stops after the 1st empty row
// All sheets are added to an ArrayList
// All rows are added to an ArrayList in the sheets ArrayList
// All cell values are added to an ArrayList in the rows ArrayList
// Formulas in cells are evaluated
// Excel values can have only 3 types: String, boolean, double
public ArrayList<ArrayList<ArrayList<Object>>> parseExcelBook(String excelPath) {
// All values are returned as String
private ArrayList<ArrayList<ArrayList<String>>> parseExcelBook(String excelPath) {
try (FileInputStream fileInputStream = new FileInputStream(excelPath)) {
Workbook workBook = new XSSFWorkbook(fileInputStream);
return parseBook(workBook);
@@ -33,9 +34,13 @@ public class FlowableExcelParser {
}
}
public ArrayList<ArrayList<ArrayList<String>>> parseExcelBookFromResource(String excelResourcePath) {
return parseExcelBook(getResourcePath(excelResourcePath));
}
@Deprecated
public ArrayList<ArrayList<Object>> parseExcelSheet(String excelPath) {
try (FileInputStream fileInputStream = new FileInputStream(excelPath)) {
public ArrayList<ArrayList<String>> parseExcelSheetFromResource(String excelResourcePath) {
try (FileInputStream fileInputStream = new FileInputStream(getResourcePath(excelResourcePath))) {
Workbook workBook = new XSSFWorkbook(fileInputStream);
FormulaEvaluator formulaEvaluator = workBook.getCreationHelper().createFormulaEvaluator();
Sheet workSheet = workBook.getSheetAt(0);
@@ -45,8 +50,17 @@ public class FlowableExcelParser {
}
}
protected ArrayList<ArrayList<ArrayList<Object>>> parseBook(Workbook workBook ) {
ArrayList<ArrayList<ArrayList<Object>>> book = new ArrayList<>();
private String getResourcePath(String resourcePath) {
ClassLoader classLoader = FlowableExcelParser.class.getClassLoader();
URL resourceUrl = classLoader.getResource(resourcePath);
if (resourceUrl == null) {
throw new RuntimeException("Resource not found: " + resourcePath);
}
return resourceUrl.getPath();
}
private ArrayList<ArrayList<ArrayList<String>>> parseBook(Workbook workBook ) {
ArrayList<ArrayList<ArrayList<String>>> book = new ArrayList<>();
FormulaEvaluator formulaEvaluator = workBook.getCreationHelper().createFormulaEvaluator();
for (Sheet workSheet : workBook) {
book.add(parseSheet(workSheet, formulaEvaluator));
@@ -54,14 +68,15 @@ public class FlowableExcelParser {
return book;
}
protected ArrayList<ArrayList<Object>> parseSheet(Sheet workSheet, FormulaEvaluator formulaEvaluator) {
ArrayList<ArrayList<Object>> sheet = new ArrayList<>();
private ArrayList<ArrayList<String>> parseSheet(Sheet workSheet, FormulaEvaluator formulaEvaluator) {
ArrayList<ArrayList<String>> sheet = new ArrayList<>();
// Count columns of first row
int maxCols = getMaxCols(workSheet.getRow(0));
int lastRowNum = workSheet.getLastRowNum();
DataFormatter dataFormatter = new DataFormatter();
for (int rowIndex = 0; rowIndex < lastRowNum; rowIndex++) {
Row workRow = workSheet.getRow(rowIndex);
ArrayList<Object> row = parseRow(workRow, maxCols, formulaEvaluator);
ArrayList<String> row = parseRow(workRow, maxCols, formulaEvaluator, dataFormatter);
if (row == null) {
// Take all rows until the first empty row
break;
@@ -72,34 +87,19 @@ public class FlowableExcelParser {
return sheet;
}
protected ArrayList<Object> parseRow(Row workRow, int maxCols, FormulaEvaluator formulaEvaluator) {
private ArrayList<String> parseRow(Row workRow, int maxCols, FormulaEvaluator formulaEvaluator, DataFormatter dataFormatter) {
if (workRow == null) {
// Row is empty
return null;
}
ArrayList<Object> row = new ArrayList<>();
ArrayList<String> row = new ArrayList<>();
boolean allCellsNull = true;
for (int colIndex = 0; colIndex < maxCols; colIndex++) {
Object content = null;
Cell workCell = workRow.getCell(colIndex);
if (workCell != null) {
CellType workCellType = workCell.getCellType();
if (workCellType != CellType.BLANK && workCellType != CellType._NONE) {
formulaEvaluator.evaluate(workCell);
String content = dataFormatter.formatCellValue(workCell, formulaEvaluator);
if (content != null && !content.isEmpty()) {
allCellsNull = false;
switch (workCellType) {
case STRING : content = workCell.getStringCellValue(); break;
case BOOLEAN : content = workCell.getBooleanCellValue(); break;
case NUMERIC : content = workCell.getNumericCellValue(); break;
case FORMULA :
CellValue evaluatedValue = formulaEvaluator.evaluate(workCell);
switch (evaluatedValue.getCellType()) {
case STRING : content = evaluatedValue.getStringValue(); break;
case BOOLEAN : content = evaluatedValue.getBooleanValue(); break;
case NUMERIC : content = evaluatedValue.getNumberValue(); break;
}
break;
}
}
}
row.add(content);
}
@@ -111,7 +111,7 @@ public class FlowableExcelParser {
}
}
protected int getMaxCols(Row workRow) {
private int getMaxCols(Row workRow) {
int maxCol = workRow.getLastCellNum();
for (int colIndex = 0; colIndex < maxCol; colIndex++) {
Cell workCell = workRow.getCell(colIndex);
@@ -124,15 +124,15 @@ public class FlowableExcelParser {
}
@Deprecated
public ArrayList<LinkedHashMap<String, Object>> parseExcel(String resourcePath) {
try (FileInputStream fileInputStream = new FileInputStream(resourcePath)) {
public ArrayList<LinkedHashMap<String, Object>> parseExcelFromResource(String resourcePath) {
try (FileInputStream fileInputStream = new FileInputStream(getResourcePath(resourcePath))) {
Workbook workBook = new XSSFWorkbook(fileInputStream);
FormulaEvaluator formulaEvaluator = workBook.getCreationHelper().createFormulaEvaluator();
ArrayList<ArrayList<Object>> sheet = parseSheet(workBook.getSheetAt(0), formulaEvaluator);
ArrayList<ArrayList<String>> sheet = parseSheet(workBook.getSheetAt(0), formulaEvaluator);
ArrayList<String> colHeaders = new ArrayList<>();
ArrayList<LinkedHashMap<String, Object>> rowList = new ArrayList<>();
for (int rowIndex = 0; rowIndex < sheet.size(); rowIndex++) {
ArrayList<Object> row = sheet.get(rowIndex);
ArrayList<String> row = sheet.get(rowIndex);
if (rowIndex == 0) {
// Header row
for (Object cell : row) {
@@ -8,7 +8,6 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.Map;
@SpringBootApplication
@@ -29,12 +28,8 @@ public class ParserApp {
throw new RuntimeException(e);
}
URL resourceUrl = ParserApp.class.getClassLoader().getResource("flowableExcelParserExampleData.xlsx");
assert resourceUrl != null;
logger.info("parsed excel={}", flowableExcelParser.parseExcelBook(resourceUrl.getPath()));
logger.info("parsed excel={}", flowableExcelParser.parseExcelBookFromResource("flowableExcelParserExampleData.xlsx"));
resourceUrl = ParserApp.class.getClassLoader().getResource("flowableExcelMapperExampleData.xlsx");
assert resourceUrl != null;
logger.info("mapped excel={}", flowableExcelMapper.excelBookToJsonNode(resourceUrl.getPath()));
logger.info("mapped excel={}", flowableExcelMapper.excelBookResourceToJsonNode("flowableExcelMapperExampleData.xlsx"));
}
}
@@ -1,8 +1,6 @@
package com.example.parser;
import com.fasterxml.jackson.databind.JsonNode;
import org.assertj.core.api.Assertions;
import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
@@ -12,7 +10,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import java.net.URL;
import java.util.ArrayList;
import java.util.stream.Stream;
@@ -27,16 +24,9 @@ class FlowableExcelMapperTest {
private static final Logger logger = LoggerFactory.getLogger(FlowableExcelMapperTest.class);
protected String getResourcePath(String path) {
ClassLoader classLoader = FlowableExcelMapperTest.class.getClassLoader();
URL resourceUrl = classLoader.getResource(path);
Assertions.assertThat(resourceUrl).isNotNull();
return resourceUrl.getPath();
}
protected Stream<Arguments> getArgumentsFromExcel(String path) {
ArrayList<Arguments> argumentList = new ArrayList<>();
JsonNode book = flowableExcelMapper.excelBookToJsonNode(getResourcePath(path));
JsonNode book = flowableExcelMapper.excelBookResourceToJsonNode(path);
for (JsonNode sheet : book) {
for (JsonNode row : sheet) {
argumentList.add(Arguments.of(row));
@@ -45,7 +35,6 @@ class FlowableExcelMapperTest {
return argumentList.stream();
}
@NotNull
protected Stream<Arguments> excelMapperTestData() {
return getArgumentsFromExcel("flowableExcelMapperTestData.xlsx");
}
@@ -9,7 +9,6 @@ import org.junit.jupiter.params.provider.MethodSource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import java.net.URL;
import java.util.*;
import java.util.stream.Stream;
@@ -21,23 +20,16 @@ class FlowableExcelParserTest {
@Autowired
private FlowableExcelParser flowableExcelParser;
protected String getResourcePath(String path) {
ClassLoader classLoader = FlowableExcelParserTest.class.getClassLoader();
URL resourceUrl = classLoader.getResource(path);
Assertions.assertThat(resourceUrl).isNotNull();
return resourceUrl.getPath();
}
protected Stream<Arguments> getRowsFromExcel(String path) {
protected Stream<Arguments> getArgumentsFromExcelBook(String resourcePath) {
ArrayList<Arguments> argumentList = new ArrayList<>();
ArrayList<ArrayList<ArrayList<Object>>> book = flowableExcelParser.parseExcelBook(getResourcePath(path));
for (ArrayList<ArrayList<Object>> sheet : book) {
for (ArrayList<Object> row : sheet) {
ArrayList<ArrayList<ArrayList<String>>> book = flowableExcelParser.parseExcelBookFromResource(resourcePath);
for (ArrayList<ArrayList<String>> sheet : book) {
for (ArrayList<String> row : sheet) {
argumentList.add(Arguments.of(row));
}
}
ArrayList<ArrayList<Object>> sheet = flowableExcelParser.parseExcelSheet(getResourcePath(path));
for (ArrayList<Object> row : sheet) {
ArrayList<ArrayList<String>> sheet = flowableExcelParser.parseExcelSheetFromResource(resourcePath);
for (ArrayList<String> row : sheet) {
argumentList.add(Arguments.of(row));
}
return argumentList.stream();
@@ -45,21 +37,21 @@ class FlowableExcelParserTest {
@NotNull
protected Stream<Arguments> booleanExcelParserTestData() {
return getRowsFromExcel("flowableExcelParserTestDataBoolean.xlsx");
return getArgumentsFromExcelBook("flowableExcelParserTestDataBoolean.xlsx");
}
@ParameterizedTest
@MethodSource("booleanExcelParserTestData")
public void booleanExcelParserTest(ArrayList<Object> row) {
Boolean valOr = (Boolean) row.get(0) || (Boolean) row.get(1) || (Boolean) row.get(2);
Assertions.assertThat(row.get(3)).isEqualTo(valOr);
Boolean valOr = Boolean.parseBoolean(row.get(0).toString()) || Boolean.parseBoolean(row.get(1).toString()) || Boolean.parseBoolean(row.get(2).toString());
Assertions.assertThat(Boolean.parseBoolean(row.get(3).toString())).isEqualTo(valOr);
Boolean valAnd = (Boolean) row.get(0) && (Boolean) row.get(1) && (Boolean) row.get(2);
Assertions.assertThat(row.get(4)).isEqualTo(valAnd);
Boolean valAnd = Boolean.parseBoolean(row.get(0).toString()) && Boolean.parseBoolean(row.get(1).toString()) && Boolean.parseBoolean(row.get(2).toString());
Assertions.assertThat(Boolean.parseBoolean(row.get(4).toString())).isEqualTo(valAnd);
}
@NotNull
protected Stream<Arguments> stringExcelParserTestData() {
return getRowsFromExcel("flowableExcelParserTestDataString.xlsx");
return getArgumentsFromExcelBook("flowableExcelParserTestDataString.xlsx");
}
@ParameterizedTest
@MethodSource("stringExcelParserTestData")
@@ -76,25 +68,25 @@ class FlowableExcelParserTest {
@NotNull
protected Stream<Arguments> numberExcelParserTestData() {
return getRowsFromExcel("flowableExcelParserTestDataInteger.xlsx");
return getArgumentsFromExcelBook("flowableExcelParserTestDataInteger.xlsx");
}
@ParameterizedTest
@MethodSource("numberExcelParserTestData")
public void numberExcelParserTest(ArrayList<Object> row) {
Double val12 = (Double) row.get(0) + (Double) row.get(1);
Assertions.assertThat(row.get(3)).isEqualTo(val12);
Integer val12 = Integer.parseInt(row.get(0).toString()) + Integer.parseInt(row.get(1).toString());
Assertions.assertThat(Integer.parseInt(row.get(3).toString())).isEqualTo(val12);
Double val23 = (Double) row.get(1) + (Double) row.get(2);
Assertions.assertThat(row.get(4)).isEqualTo(val23);
Integer val23 = Integer.parseInt(row.get(1).toString()) + Integer.parseInt(row.get(2).toString());
Assertions.assertThat(Integer.parseInt(row.get(4).toString())).isEqualTo(val23);
Double val123 = (Double) row.get(0) + (Double) row.get(1) + (Double) row.get(2);
Assertions.assertThat(row.get(5)).isEqualTo(val123);
Integer val123 = Integer.parseInt(row.get(0).toString()) + Integer.parseInt(row.get(1).toString()) + Integer.parseInt(row.get(2).toString());
Assertions.assertThat(Integer.parseInt(row.get(5).toString())).isEqualTo(val123);
}
@Deprecated
protected Stream<Arguments> getArgumentsFromExcel(String path) {
protected Stream<Arguments> getArgumentsFromExcel(String resourcePath) {
ArrayList<Arguments> argumentList = new ArrayList<>();
ArrayList<LinkedHashMap<String, Object>> rowList = flowableExcelParser.parseExcel(getResourcePath(path));
ArrayList<LinkedHashMap<String, Object>> rowList = flowableExcelParser.parseExcelFromResource(resourcePath);
for (LinkedHashMap<String, Object> row : rowList) {
LinkedHashMap<String, Object> inputMap = new LinkedHashMap<>();
LinkedHashMap<String, Object> outputMap = new LinkedHashMap<>();
@@ -122,12 +114,12 @@ class FlowableExcelParserTest {
for (Map.Entry<String, Object> entry : outputMap.entrySet()) {
String key = entry.getKey();
if (key.equals("output1")) {
Boolean val = (Boolean)inputMap.get("input1") || (Boolean)inputMap.get("input2") || (Boolean)inputMap.get("input3");
Assertions.assertThat(entry.getValue()).isEqualTo(val);
Boolean val = Boolean.parseBoolean(inputMap.get("input1").toString()) || Boolean.parseBoolean(inputMap.get("input2").toString()) || Boolean.parseBoolean(inputMap.get("input3").toString());
Assertions.assertThat(Boolean.parseBoolean(entry.getValue().toString())).isEqualTo(val);
}
if (key.equals("output2")) {
Boolean val = (Boolean)inputMap.get("input1") && (Boolean)inputMap.get("input2") && (Boolean)inputMap.get("input3");
Assertions.assertThat(entry.getValue()).isEqualTo(val);
Boolean val = Boolean.parseBoolean(inputMap.get("input1").toString()) && Boolean.parseBoolean(inputMap.get("input2").toString()) && Boolean.parseBoolean(inputMap.get("input3").toString());
Assertions.assertThat(Boolean.parseBoolean(entry.getValue().toString())).isEqualTo(val);
}
}
}
@@ -166,16 +158,16 @@ class FlowableExcelParserTest {
for (Map.Entry<String, Object> entry : outputMap.entrySet()) {
String key = entry.getKey();
if (key.equals("output1")) {
Double val = (Double)inputMap.get("input1") + (Double)inputMap.get("input2");
Assertions.assertThat(entry.getValue()).isEqualTo(val);
Integer val12 = Integer.parseInt(inputMap.get("input1").toString()) + Integer.parseInt(inputMap.get("input2").toString());
Assertions.assertThat(Integer.parseInt(entry.getValue().toString())).isEqualTo(val12);
}
if (key.equals("output2")) {
Double val = (Double)inputMap.get("input2") + (Double)inputMap.get("input3");
Assertions.assertThat(entry.getValue()).isEqualTo(val);
Integer val23 = Integer.parseInt(inputMap.get("input2").toString()) + Integer.parseInt(inputMap.get("input3").toString());
Assertions.assertThat(Integer.parseInt(entry.getValue().toString())).isEqualTo(val23);
}
if (key.equals("output3")) {
Double val = (Double)inputMap.get("input1") + (Double)inputMap.get("input2") + (Double)inputMap.get("input3");
Assertions.assertThat(entry.getValue()).isEqualTo(val);
Integer val12 = Integer.parseInt(inputMap.get("input1").toString()) + Integer.parseInt(inputMap.get("input2").toString()) + Integer.parseInt(inputMap.get("input3").toString());
Assertions.assertThat(Integer.parseInt(entry.getValue().toString())).isEqualTo(val12);
}
}
}
Binary file not shown.