FREEMARKER-144 Proof Of Concept for providing DataFrames (#14)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b610dbf..045191a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@
## 0.1.0-SNAPSHOT
### Added
+* [FREEMARKER-144] Proof Of Concept for providing DataFrames
* [FREEMARKER-142] Support Transformation Of Directories
* [FREEMARKER-139] freemarker-cli: Provide GsonTool to align with Maven plugin
* Environment variables can bes passed as `DataSource`
@@ -38,4 +39,5 @@
[FREEMARKER-136]: https://issues.apache.org/jira/browse/FREEMARKER-136
[FREEMARKER-138]: https://issues.apache.org/jira/browse/FREEMARKER-138
[FREEMARKER-139]: https://issues.apache.org/jira/browse/FREEMARKER-139
-[FREEMARKER-142]: https://issues.apache.org/jira/browse/FREEMARKER-142
\ No newline at end of file
+[FREEMARKER-142]: https://issues.apache.org/jira/browse/FREEMARKER-142
+[FREEMARKER-144]: https://issues.apache.org/jira/browse/FREEMARKER-144
\ No newline at end of file
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java
new file mode 100644
index 0000000..1abf21b
--- /dev/null
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.freemarker.generator.base.table;
+
+import org.apache.freemarker.generator.base.util.ListUtils;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyList;
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Simple table model filled from maps or rows representing tabular data.
+ */
+public class Table {
+
+ /** List of column names */
+ private final List<String> columnNames;
+
+ /** Column types derived from tabular data */
+ private final List<Class<?>> columnTypes;
+
+ /** Table data as rows */
+ private final List<List<Object>> values;
+
+ /** Map column names to column index */
+ private final Map<String, Integer> columnMap;
+
+ private Table() {
+ this.columnNames = emptyList();
+ this.columnTypes = emptyList();
+ this.values = emptyList();
+ this.columnMap = new HashMap<>();
+ }
+
+ private Table(Collection<String> columnNames, Collection<Class<?>> columnTypes, List<List<Object>> columnValuesList) {
+ this.columnNames = new ArrayList<>(requireNonNull(columnNames));
+ this.columnTypes = new ArrayList<>(requireNonNull(columnTypes));
+ this.values = ListUtils.transpose(requireNonNull(columnValuesList));
+ this.columnMap = columnMap(this.columnNames);
+ }
+
+ public List<String> getColumnNames() {
+ return columnNames;
+ }
+
+ public List<Class<?>> getColumnTypes() {
+ return columnTypes;
+ }
+
+ public List<List<Object>> getValues() {
+ return values;
+ }
+
+ public int getNrOfColumns() {
+ return columnNames.isEmpty() ? values.isEmpty() ? 0 : values.get(0).size() : columnNames.size();
+ }
+
+ public int size() {
+ return values.size();
+ }
+
+ public boolean isEmpty() {
+ return values.isEmpty();
+ }
+
+ public List<Object> getRow(int row) {
+ return values.get(row);
+ }
+
+ public Object get(int row, int column) {
+ return values.get(row).get(column);
+ }
+
+ public Object get(int row, String column) {
+ return values.get(row).get(columnMap.get(column));
+ }
+
+ public boolean hasColumnHeaderRow() {
+ return !columnNames.isEmpty();
+ }
+
+ /**
+ * Create a table from a list of maps. Non-tabular data is supported,
+ * i.e. not all maps contains all possible keys.
+ *
+ * @param maps list of maps
+ * @return table
+ */
+ public static Table fromMaps(Collection<Map<String, Object>> maps) {
+ if (maps == null || maps.isEmpty()) {
+ return new Table();
+ }
+
+ final List<String> columnNames = columnNames(maps);
+ final List<List<Object>> columnValuesList = columnValuesList(maps, columnNames);
+ final List<Class<?>> columnTypes = columnTypes(columnValuesList);
+
+ return new Table(
+ columnNames,
+ columnTypes,
+ columnValuesList);
+ }
+
+ /**
+ * Create a table from a list of rows representing tabular data.
+ *
+ * @param rows row values
+ * @return table
+ */
+ public static Table fromRows(List<List<Object>> rows) {
+ requireNonNull(rows, "rows is null");
+
+ final List<List<Object>> columnValuesList = ListUtils.transpose(rows);
+ final List<Class<?>> columnTypes = columnTypes(columnValuesList);
+
+ return new Table(
+ new ArrayList<>(),
+ columnTypes,
+ columnValuesList);
+ }
+
+ /**
+ * Create a table from a list of rows representing tabular data
+ * where the first row may consists of column headers.
+ *
+ * @param rows row values
+ * @param withFirstRowAsColumnNames column names as first row?
+ * @return table
+ */
+ public static Table fromRows(List<List<Object>> rows, boolean withFirstRowAsColumnNames) {
+ if (ListUtils.isNullOrEmpty(rows) && withFirstRowAsColumnNames) {
+ throw new IllegalArgumentException("Header columns expected but list is empty");
+ }
+
+ if (withFirstRowAsColumnNames) {
+ final List<String> columnNames = columnNames(rows.get(0));
+ final List<List<Object>> table = rows.subList(1, rows.size());
+ return fromRows(columnNames, table);
+ } else {
+ return fromRows(rows);
+ }
+ }
+
+ /**
+ * Create a table from column names and row values.
+ *
+ * @param columnNames list of column names
+ * @param rows row values as rows
+ * @return table
+ */
+ public static Table fromRows(Collection<String> columnNames, List<List<Object>> rows) {
+ requireNonNull(columnNames, "columnNames is null");
+ requireNonNull(rows, "rows is null");
+
+ final List<List<Object>> columnValuesList = ListUtils.transpose(rows);
+ final List<Class<?>> columnTypes = columnTypes(columnValuesList);
+
+ return new Table(
+ new ArrayList<>(columnNames),
+ columnTypes,
+ columnValuesList);
+ }
+
+ /**
+ * Determine column names based on all available keys of the maps.
+ *
+ * @param maps list of maps
+ * @return column names
+ */
+ private static List<String> columnNames(Collection<Map<String, Object>> maps) {
+ return maps.stream()
+ .map(Map::keySet)
+ .flatMap(Collection::stream)
+ .distinct()
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Determine column names based on a single list.
+ *
+ * @param list list of column names
+ * @return column names
+ */
+ private static List<String> columnNames(List<Object> list) {
+ return list.stream()
+ .map(Object::toString)
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Determine all column values.
+ *
+ * @param maps list of maps
+ * @param columnNames column names
+ * @return list of column values
+ */
+ private static List<List<Object>> columnValuesList(Collection<Map<String, Object>> maps, List<String> columnNames) {
+ return columnNames.stream()
+ .map(columnName -> columnValues(maps, columnName))
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Determine the values of a single column.
+ *
+ * @param maps list of maps
+ * @param columnName column name
+ * @return values of the given column
+ */
+ private static List<Object> columnValues(Collection<Map<String, Object>> maps, String columnName) {
+ return maps.stream()
+ .map(map -> map.getOrDefault(columnName, null))
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Determine the column types based on the first non-null values.
+ *
+ * @param columnValuesList list of column values
+ * @return classes of the first non-null value
+ */
+ private static List<Class<?>> columnTypes(List<List<Object>> columnValuesList) {
+ return columnValuesList.stream()
+ .map(Table::columnType)
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Determine the column type based on the first non-null value.
+ *
+ * @param columnValues column values
+ * @return class of the first non-null value
+ */
+ private static Class<?> columnType(List<Object> columnValues) {
+ return ListUtils.coalesce(columnValues).getClass();
+ }
+
+ private static Map<String, Integer> columnMap(List<String> columnNames) {
+ final Map<String, Integer> result = new HashMap<>();
+ for (int i = 0; i < columnNames.size(); i++) {
+ result.put(columnNames.get(i), i);
+ }
+ return result;
+ }
+}
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/template/TemplateTransformationsBuilder.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/template/TemplateTransformationsBuilder.java
index 37100aa..fe62017 100644
--- a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/template/TemplateTransformationsBuilder.java
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/template/TemplateTransformationsBuilder.java
@@ -13,7 +13,8 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- */package org.apache.freemarker.generator.base.template;
+ */
+package org.apache.freemarker.generator.base.template;
import org.apache.freemarker.generator.base.file.RecursiveFileSupplier;
import org.apache.freemarker.generator.base.util.NonClosableWriterWrapper;
@@ -22,8 +23,6 @@
import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.ArrayList;
@@ -73,7 +72,7 @@
final List<TemplateTransformation> result = new ArrayList<>();
- if (template != null) {
+ if (hasInteractiveTemplate()) {
final File outputFile = outputs.isEmpty() ? null : outputs.get(0);
result.add(resolveInteractiveTemplate(outputFile));
} else {
@@ -249,18 +248,8 @@
return excludes.isEmpty() ? null : excludes.get(0);
}
- private Writer writer(String outputFile, String outputEncoding) {
- try {
- if (writer != null) {
- return writer;
- } else if (!StringUtils.isEmpty(outputFile)) {
- return new BufferedWriter(new FileWriter(outputFile));
- } else {
- return new BufferedWriter(new OutputStreamWriter(System.out, outputEncoding));
- }
- } catch (IOException e) {
- throw new RuntimeException("Unable to create writer", e);
- }
+ private boolean hasInteractiveTemplate() {
+ return template != null;
}
private static File getTemplateOutputFile(File templateDirectory, File templateFile, File outputDirectory) {
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ListUtils.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ListUtils.java
new file mode 100644
index 0000000..e04d16a
--- /dev/null
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ListUtils.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.freemarker.generator.base.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+public class ListUtils {
+
+ public static <T> boolean isNullOrEmpty(final List<T> list) {
+ return list == null || list.isEmpty();
+ }
+
+ /**
+ * Transposes the given tabular data, swapping rows with columns.
+ *
+ * @param <T> the type of the table
+ * @param table the table
+ * @return the transposed table
+ * @throws NullPointerException if the given table is {@code null}
+ */
+ public static <T> List<List<T>> transpose(final List<List<T>> table) {
+ if (isNullOrEmpty(table)) {
+ return new ArrayList<>();
+ }
+
+ final List<List<T>> result = new ArrayList<>();
+ for (int i = 0; i < table.get(0).size(); i++) {
+ final List<T> col = new ArrayList<>();
+ for (List<T> row : table) {
+ col.add(row.get(i));
+ }
+ result.add(col);
+ }
+ return result;
+ }
+
+ /**
+ * Returns the first non-null value of the list.
+ *
+ * @param list array
+ * @param <T> the type of the array
+ * @return copied array
+ */
+ public static <T> T coalesce(List<T> list) {
+ return list.stream().filter(Objects::nonNull).findFirst().orElseGet(() -> null);
+ }
+
+ /**
+ * Copy an array to another array while casting to <code>R</code>.
+ *
+ * @param array array to copy
+ * @param <T> the source type of the array
+ * @param <R> the target type of the array
+ * @return copied array
+ */
+ @SuppressWarnings("unchecked")
+ public static <T, R> List<R> copy(final List<T> array) {
+ final List<R> result = new ArrayList<>();
+ for (int i = 0; i < array.size(); i++) {
+ result.set(i, (R) array.get(i));
+ }
+ return result;
+ }
+}
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/MapBuilder.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/MapBuilder.java
new file mode 100644
index 0000000..60daa80
--- /dev/null
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/MapBuilder.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.freemarker.generator.base.util;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class MapBuilder {
+
+ public static Map<String, Object> toLinkedMap(Object... data) {
+
+ final HashMap<String, Object> map = new LinkedHashMap<>();
+
+ if (data.length % 2 != 0) {
+ throw new IllegalArgumentException("Odd number of arguments");
+ }
+
+ String currKey = null;
+ int step = -1;
+
+ for (Object value : data) {
+ step++;
+ switch (step % 2) {
+ case 0:
+ if (value == null) {
+ throw new IllegalArgumentException("Null key value");
+ }
+ currKey = value.toString();
+ continue;
+ case 1:
+ map.put(currKey, value);
+ break;
+ }
+ }
+
+ return map;
+ }
+}
diff --git a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/table/TableTest.java b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/table/TableTest.java
new file mode 100644
index 0000000..bd44efc
--- /dev/null
+++ b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/table/TableTest.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.freemarker.generator.table;
+
+import org.apache.freemarker.generator.base.table.Table;
+import org.apache.freemarker.generator.base.util.MapBuilder;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertNull;
+
+public class TableTest {
+
+ public final List<Map<String, Object>> booksMaps = Arrays.asList(
+ MapBuilder.toLinkedMap(
+ "Book ID", "1",
+ "Book Name", "Computer Architecture",
+ "Category", "Computers",
+ "In Stock", true,
+ "Price", 125.60),
+ MapBuilder.toLinkedMap(
+ "Book ID", "2",
+ "Book Name", "Asp.Net 4 Blue Book",
+ "Category", "Programming",
+ "In Stock", null,
+ "Price", 56),
+ MapBuilder.toLinkedMap(
+ "Book ID", "3",
+ "Book Name", "Popular Science",
+ "Category", "Science",
+ "Price", 210.40)
+ );
+
+ public final List<String> booksHeader = Arrays.asList(
+ "Book ID",
+ "Book Name",
+ "Category",
+ "In Stock",
+ "Price");
+
+
+ public final List<List<Object>> booksList = Arrays.asList(
+ Arrays.asList("1", "Computer Architecture", "Computers", true, 125.60),
+ Arrays.asList("2", "Asp.Net 4 Blue Book", "Programming", null, 56),
+ Arrays.asList("3", "Popular Science", "Science", null, 210.40)
+ );
+
+ public final List<List<Object>> booksListWithHeaders = Arrays.asList(
+ Arrays.asList("Book ID", "Book Name", "Category", "In Stock", "Price"),
+ Arrays.asList("1", "Computer Architecture", "Computers", true, 125.60),
+ Arrays.asList("2", "Asp.Net 4 Blue Book", "Programming", null, 56),
+ Arrays.asList("3", "Popular Science", "Science", null, 210.40)
+ );
+
+ @Test
+ public void shouldConvertFromMaps() {
+ final Table table = Table.fromMaps(booksMaps);
+
+ validateBooks(table);
+ assertEquals(booksHeader, table.getColumnNames());
+ }
+
+ @Test
+ public void shouldConvertFromEmptyMaps() {
+ final Table table = Table.fromMaps(new ArrayList<>());
+
+ assertEquals(0, table.getNrOfColumns());
+ assertEquals(0, table.size());
+ }
+
+ @Test
+ public void shouldConvertFromNullMap() {
+ final Table table = Table.fromMaps(null);
+
+ assertEquals(0, table.getNrOfColumns());
+ assertEquals(0, table.size());
+ }
+
+ @Test
+ public void shouldConvertFromListsWithExplicitHeaders() {
+ final Table table = Table.fromRows(booksHeader, booksList);
+
+ validateBooks(table);
+ assertEquals(booksHeader, table.getColumnNames());
+ }
+
+ @Test
+ public void shouldConvertFromListsWithImplicitHeaders() {
+ final Table table = Table.fromRows(booksListWithHeaders, true);
+
+ validateBooks(table);
+ assertEquals(booksHeader, table.getColumnNames());
+ }
+
+ @Test
+ public void shouldConvertFromListsWithEmptyHeaders() {
+ final Table table = Table.fromRows(booksList);
+
+ validateBooks(table);
+ }
+
+ public void validateBooks(Table table) {
+ assertEquals(5, table.getNrOfColumns());
+ assertEquals(3, table.size());
+ // Book Id
+ assertEquals("1", table.get(0, 0));
+ assertEquals("2", table.get(1, 0));
+ assertEquals("3", table.get(2, 0));
+ // Book Name
+ assertEquals("Computer Architecture", table.get(0, 1));
+ assertEquals("Asp.Net 4 Blue Book", table.get(1, 1));
+ assertEquals("Popular Science", table.get(2, 1));
+ // Category
+ assertEquals("Computers", table.get(0, 2));
+ assertEquals("Programming", table.get(1, 2));
+ assertEquals("Science", table.get(2, 2));
+ // In Stock
+ assertEquals(true, table.get(0, 3));
+ assertNull(table.get(1, 3));
+ assertNull(table.get(2, 3));
+ // Price
+ assertEquals(125.60, table.get(0, 4));
+ assertEquals(56, table.get(1, 4));
+ assertEquals(210.40, table.get(2, 4));
+ }
+}
diff --git a/freemarker-generator-cli/pom.xml b/freemarker-generator-cli/pom.xml
index 8b74a0d..892473d 100644
--- a/freemarker-generator-cli/pom.xml
+++ b/freemarker-generator-cli/pom.xml
@@ -146,7 +146,7 @@
<exclude>src/main/resources/patterns/*</exclude>
<exclude>site/sample/*/**</exclude>
<exclude>src/test/data/encoding/utf8.txt</exclude>
- <exclude>src/test/data/json/environments.json</exclude>
+ <exclude>src/test/data/json/*/**</exclude>
<exclude>src/test/data/yaml/environments.yaml</exclude>
</excludes>
</configuration>
diff --git a/freemarker-generator-cli/run-samples.sh b/freemarker-generator-cli/run-samples.sh
index d760145..ec2733a 100755
--- a/freemarker-generator-cli/run-samples.sh
+++ b/freemarker-generator-cli/run-samples.sh
@@ -51,6 +51,7 @@
$FREEMARKER_CMD -i '${JsoupTool.parse(DataSources.first).select("a")[0]}' site/sample/html/dependencies.html > target/out/interactive-html.txt || { echo >&2 "Test failed. Aborting."; exit 1; }
$FREEMARKER_CMD -i '${GsonTool.toJson(YamlTool.parse(DataSources.get(0)))}' site/sample/yaml/swagger-spec.yaml > target/out/interactive-swagger.json || { echo >&2 "Test failed. Aborting."; exit 1; }
$FREEMARKER_CMD -i '${YamlTool.toYaml(GsonTool.parse(DataSources.get(0)))}' site/sample/json/swagger-spec.json > target/out/interactive-swagger.yaml || { echo >&2 "Test failed. Aborting."; exit 1; }
+$FREEMARKER_CMD -i '${DataFrameTool.print(DataFrameTool.fromMaps(GsonTool.parse(DataSources.get(0))))}' site/sample/json/github-users.json > target/out/interactive-dataframe.txt || { echo >&2 "Test failed. Aborting."; exit 1; }
#############################################################################
# CSV
@@ -101,6 +102,13 @@
fi
#############################################################################
+# DataFrame
+#############################################################################
+
+echo "templates/dataframe/example.ftl"
+$FREEMARKER_CMD -DCSV_TOOL_DELIMITER=SEMICOLON -DCSV_TOOL_HEADERS=true -t templates/dataframe/example.ftl site/sample/csv/dataframe.csv > target/out/dataframe.txt || { echo >&2 "Test failed. Aborting."; exit 1; }
+
+#############################################################################
# Grok
#############################################################################
@@ -111,6 +119,9 @@
# Excel
#############################################################################
+echo "templates/excel/dataframe/transform.ftl"
+$FREEMARKER_CMD -t templates/excel/dataframe/transform.ftl site/sample/excel/test.xls > target/out/test.xls.dataframe.txt || { echo >&2 "Test failed. Aborting."; exit 1; }
+
echo "templates/excel/html/transform.ftl"
$FREEMARKER_CMD -t templates/excel/html/transform.ftl site/sample/excel/test.xls > target/out/test.xls.html || { echo >&2 "Test failed. Aborting."; exit 1; }
$FREEMARKER_CMD -t templates/excel/html/transform.ftl site/sample/excel/test.xlsx > target/out/test.xslx.html || { echo >&2 "Test failed. Aborting."; exit 1; }
diff --git a/freemarker-generator-cli/site/sample/csv/dataframe.csv b/freemarker-generator-cli/site/sample/csv/dataframe.csv
new file mode 100644
index 0000000..b841ef9
--- /dev/null
+++ b/freemarker-generator-cli/site/sample/csv/dataframe.csv
@@ -0,0 +1,10 @@
+name;age;country
+Schmitt;24;Germany
+Parker;45;USA
+Meier;20;Germany
+Schmitt;30;France
+Peter;44;Germany
+Meier;24;Germany
+Green;33;UK
+Schmitt;30;Germany
+Meier;30;Germany
\ No newline at end of file
diff --git a/freemarker-generator-cli/site/template/application.properties b/freemarker-generator-cli/site/template/application.properties
index d5f2114..fe83a14 100644
--- a/freemarker-generator-cli/site/template/application.properties
+++ b/freemarker-generator-cli/site/template/application.properties
@@ -15,5 +15,5 @@
under the License.
-->
# == application.properties ==================================================
-server.name=${NGINX_HOSTNAME!"somehost"}
+server.name=${NGINX_HOSTNAME!"127.0.0.1"}
server.logs=${NGINX_LOGS!"/var/log/nginx"}
diff --git a/freemarker-generator-cli/site/template/nginx/nginx.conf.ftl b/freemarker-generator-cli/site/template/nginx/nginx.conf.ftl
index ac57019..7cb9f3d 100644
--- a/freemarker-generator-cli/site/template/nginx/nginx.conf.ftl
+++ b/freemarker-generator-cli/site/template/nginx/nginx.conf.ftl
@@ -14,10 +14,10 @@
specific language governing permissions and limitations
under the License.
-->
-# == nginx-conf =============================================================
+# == nginx-conf ==============================================================
server {
listen ${NGINX_PORT!"80"};
- server_name ${NGINX_HOSTNAME!"somehost"};
+ server_name ${NGINX_HOSTNAME!"127.0.0.1"};
root ${NGINX_WEBROOT!"/usr/share/nginx/www"};
index index.htm;
diff --git a/freemarker-generator-cli/src/main/config/freemarker-cli.properties b/freemarker-generator-cli/src/main/config/freemarker-cli.properties
index fa08255..bd9c917 100644
--- a/freemarker-generator-cli/src/main/config/freemarker-cli.properties
+++ b/freemarker-generator-cli/src/main/config/freemarker-cli.properties
@@ -25,15 +25,16 @@
# Configure FreeMarker Tools (name -> implementation class)
#############################################################################
freemarker.tools.CSVTool=org.apache.freemarker.generator.tools.commonscsv.CommonsCSVTool
-freemarker.tools.ExecTool=org.apache.freemarker.generator.tools.commonsexec.CommonsExecTool
+freemarker.tools.DataFrameTool=org.apache.freemarker.generator.tools.dataframe.DataFrameTool
freemarker.tools.ExcelTool=org.apache.freemarker.generator.tools.excel.ExcelTool
+freemarker.tools.ExecTool=org.apache.freemarker.generator.tools.commonsexec.CommonsExecTool
freemarker.tools.FreeMarkerTool=org.apache.freemarker.generator.tools.freemarker.FreeMarkerTool
freemarker.tools.GrokTool=org.apache.freemarker.generator.tools.grok.GrokTool
freemarker.tools.GsonTool=org.apache.freemarker.generator.tools.gson.GsonTool
freemarker.tools.JsonPathTool=org.apache.freemarker.generator.tools.jsonpath.JsonPathTool
freemarker.tools.JsoupTool=org.apache.freemarker.generator.tools.jsoup.JsoupTool
freemarker.tools.PropertiesTool=org.apache.freemarker.generator.tools.properties.PropertiesTool
-freemarker.tools.YamlTool=org.apache.freemarker.generator.tools.snakeyaml.SnakeYamlTool
freemarker.tools.SystemTool=org.apache.freemarker.generator.tools.system.SystemTool
freemarker.tools.UUIDTool=org.apache.freemarker.generator.tools.uuid.UUIDTool
freemarker.tools.XmlTool=org.apache.freemarker.generator.tools.xml.XmlTool
+freemarker.tools.YamlTool=org.apache.freemarker.generator.tools.snakeyaml.SnakeYamlTool
diff --git a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelSupplier.java b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelSupplier.java
index d8e3491..79d94b0 100644
--- a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelSupplier.java
+++ b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelSupplier.java
@@ -30,13 +30,14 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.function.Supplier;
+import java.util.stream.Collectors;
import static java.util.Objects.requireNonNull;
-import static java.util.stream.Collectors.toMap;
import static org.apache.freemarker.generator.base.mime.Mimetypes.MIME_APPLICATION_JSON;
import static org.apache.freemarker.generator.base.mime.Mimetypes.MIME_TEXT_PLAIN;
import static org.apache.freemarker.generator.base.mime.Mimetypes.MIME_TEXT_YAML;
@@ -62,12 +63,12 @@
public Map<String, Object> get() {
return sources.stream()
.filter(StringUtils::isNotEmpty)
- .map(this::toDataModel)
+ .map(DataModelSupplier::toDataModel)
.flatMap(map -> map.entrySet().stream())
- .collect(toMap(Entry::getKey, Entry::getValue));
+ .collect(Collectors.toMap(Entry::getKey, Entry::getValue));
}
- protected Map<String, Object> toDataModel(String source) {
+ private static Map<String, Object> toDataModel(String source) {
final NamedUri namedUri = NamedUriStringParser.parse(source);
final DataSource dataSource = DataSourceFactory.fromNamedUri(namedUri);
final boolean isExplodedDataModel = !namedUri.hasName();
@@ -84,19 +85,19 @@
}
}
- private Map<String, Object> fromJson(DataSource dataSource, boolean isExplodedDataModel) {
+ private static Map<String, Object> fromJson(DataSource dataSource, boolean isExplodedDataModel) {
final GsonTool gsonTool = new GsonTool();
- final Map<String, Object> map = gsonTool.parse(dataSource);
- return fromMap(dataSource.getName(), map, isExplodedDataModel);
+ final Object json = gsonTool.parse(dataSource);
+ return toMap(dataSource.getName(), json, isExplodedDataModel);
}
- private Map<String, Object> fromYaml(DataSource dataSource, boolean isExplodedDataModel) {
+ private static Map<String, Object> fromYaml(DataSource dataSource, boolean isExplodedDataModel) {
final SnakeYamlTool snakeYamlTool = new SnakeYamlTool();
- final Map<String, Object> map = snakeYamlTool.parse(dataSource);
- return fromMap(dataSource.getName(), map, isExplodedDataModel);
+ final Object yaml = snakeYamlTool.parse(dataSource);
+ return toMap(dataSource.getName(), yaml, isExplodedDataModel);
}
- private Map<String, Object> fromProperties(DataSource dataSource, boolean isExplodedDataModel) {
+ private static Map<String, Object> fromProperties(DataSource dataSource, boolean isExplodedDataModel) {
final Map<String, Object> result = new HashMap<>();
final URI uri = dataSource.getUri();
@@ -114,13 +115,27 @@
return result;
}
- private Map<String, Object> fromMap(String name, Map<String, Object> map, boolean isExplodedDataModel) {
+ @SuppressWarnings("unchecked")
+ private static Map<String, Object> toMap(String name, Object obj, boolean isExplodedDataModel) {
final Map<String, Object> result = new HashMap<>();
- if (isExplodedDataModel) {
- map.forEach(result::put);
- } else {
- result.put(name, map);
+ if (obj instanceof Map) {
+ final Map<String, Object> map = (Map<String, Object>) obj;
+ if (isExplodedDataModel) {
+ map.forEach(result::put);
+ } else {
+ result.put(name, map);
+ }
+ } else if (obj instanceof List) {
+ final List<Object> list = (List<Object>) obj;
+ if (isExplodedDataModel) {
+ for (Object entry : list) {
+ final Map<String, Object> map = (Map<String, Object>) entry;
+ map.forEach(result::put);
+ }
+ } else {
+ result.put(name, list);
+ }
}
return result;
diff --git a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/Suppliers.java b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/Suppliers.java
index 2c220b2..7151600 100644
--- a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/Suppliers.java
+++ b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/Suppliers.java
@@ -68,14 +68,14 @@
}
public static TemplateTransformationsSupplier templateTransformationsSupplier(Settings settings) {
- return (() -> TemplateTransformationsBuilder.builder()
+ return () -> TemplateTransformationsBuilder.builder()
.setTemplate("interactive", settings.getInteractiveTemplate())
.addSources(settings.getTemplates())
.addInclude(settings.getTemplateFileIncludePattern())
.addExclude(settings.getTemplateFileExcludePattern())
.addOutput(settings.getOutput())
.setWriter(settings.getWriter())
- .build());
+ .build();
}
public static PropertiesSupplier propertiesSupplier(String fileName) {
diff --git a/freemarker-generator-cli/src/main/resources/freemarker-cli.properties b/freemarker-generator-cli/src/main/resources/freemarker-cli.properties
index 0a0542a..bd9c917 100644
--- a/freemarker-generator-cli/src/main/resources/freemarker-cli.properties
+++ b/freemarker-generator-cli/src/main/resources/freemarker-cli.properties
@@ -25,6 +25,7 @@
# Configure FreeMarker Tools (name -> implementation class)
#############################################################################
freemarker.tools.CSVTool=org.apache.freemarker.generator.tools.commonscsv.CommonsCSVTool
+freemarker.tools.DataFrameTool=org.apache.freemarker.generator.tools.dataframe.DataFrameTool
freemarker.tools.ExcelTool=org.apache.freemarker.generator.tools.excel.ExcelTool
freemarker.tools.ExecTool=org.apache.freemarker.generator.tools.commonsexec.CommonsExecTool
freemarker.tools.FreeMarkerTool=org.apache.freemarker.generator.tools.freemarker.FreeMarkerTool
diff --git a/freemarker-generator-cli/src/main/scripts/run-samples.sh b/freemarker-generator-cli/src/main/scripts/run-samples.sh
index c005f09..780a702 100755
--- a/freemarker-generator-cli/src/main/scripts/run-samples.sh
+++ b/freemarker-generator-cli/src/main/scripts/run-samples.sh
@@ -51,6 +51,7 @@
$FREEMARKER_CMD -i '${JsoupTool.parse(DataSources.first).select("a")[0]}' site/sample/html/dependencies.html > target/out/interactive-html.txt || { echo >&2 "Test failed. Aborting."; exit 1; }
$FREEMARKER_CMD -i '${GsonTool.toJson(YamlTool.parse(DataSources.get(0)))}' site/sample/yaml/swagger-spec.yaml > target/out/interactive-swagger.json || { echo >&2 "Test failed. Aborting."; exit 1; }
$FREEMARKER_CMD -i '${YamlTool.toYaml(GsonTool.parse(DataSources.get(0)))}' site/sample/json/swagger-spec.json > target/out/interactive-swagger.yaml || { echo >&2 "Test failed. Aborting."; exit 1; }
+$FREEMARKER_CMD -i '${DataFrameTool.print(DataFrameTool.fromMaps(GsonTool.parse(DataSources.get(0))))}' site/sample/json/github-users.json > target/out/interactive-dataframe.txt || { echo >&2 "Test failed. Aborting."; exit 1; }
#############################################################################
# CSV
@@ -101,6 +102,13 @@
fi
#############################################################################
+# DataFrame
+#############################################################################
+
+echo "templates/dataframe/example.ftl"
+$FREEMARKER_CMD -DCSV_TOOL_DELIMITER=SEMICOLON -DCSV_TOOL_HEADERS=true -t templates/dataframe/example.ftl site/sample/csv/dataframe.csv > target/out/dataframe.txt || { echo >&2 "Test failed. Aborting."; exit 1; }
+
+#############################################################################
# Grok
#############################################################################
@@ -111,6 +119,9 @@
# Excel
#############################################################################
+echo "templates/excel/dataframe/transform.ftl"
+$FREEMARKER_CMD -t templates/excel/dataframe/transform.ftl site/sample/excel/test.xls > target/out/test.xls.dataframe.txt || { echo >&2 "Test failed. Aborting."; exit 1; }
+
echo "templates/excel/html/transform.ftl"
$FREEMARKER_CMD -t templates/excel/html/transform.ftl site/sample/excel/test.xls > target/out/test.xls.html || { echo >&2 "Test failed. Aborting."; exit 1; }
$FREEMARKER_CMD -t templates/excel/html/transform.ftl site/sample/excel/test.xlsx > target/out/test.xslx.html || { echo >&2 "Test failed. Aborting."; exit 1; }
diff --git a/freemarker-generator-cli/src/site/markdown/cli/tools/dataframe.md b/freemarker-generator-cli/src/site/markdown/cli/tools/dataframe.md
new file mode 100644
index 0000000..3523294
--- /dev/null
+++ b/freemarker-generator-cli/src/site/markdown/cli/tools/dataframe.md
@@ -0,0 +1,205 @@
+# DataFrameTool
+
+The `DataFrameTool` uses [nRo/DataFrame](https://github.com/nRo/DataFrame) to convert tabular data into a `DataFrame`.
+
+A `DataFrame` allows declartive filtering and transformation of tabular data, i.e. little code to write.
+
+Currently the following sources are supported
+
+* Apache Commons CSV Parser
+* JSON arrays represented as collection of maps
+* Excel sheets represented as rows
+
+## CSV Examples
+
+[nRo/DataFrame]("https://raw.githubusercontent.com/nRo/DataFrame/master/src/test/resources/users.csv") provides the following CSV file
+
+```
+┌────────────┬────────────┬────────────┐
+│#name │#age │#country │
+├────────────┼────────────┼────────────┤
+│Schmitt │24 │Germany │
+├────────────┼────────────┼────────────┤
+│Parker │45 │USA │
+├────────────┼────────────┼────────────┤
+│Meier │20 │Germany │
+├────────────┼────────────┼────────────┤
+│Schmitt │30 │France │
+├────────────┼────────────┼────────────┤
+│Peter │44 │Germany │
+├────────────┼────────────┼────────────┤
+│Meier │24 │Germany │
+├────────────┼────────────┼────────────┤
+│Green │33 │UK │
+├────────────┼────────────┼────────────┤
+│Schmitt │30 │Germany │
+├────────────┼────────────┼────────────┤
+│Meier │30 │Germany │
+└────────────┴────────────┴────────────┘
+```
+
+and create a `DateFrame` using the following code
+
+```
+<#assign cvsFormat = CSVTool.formats["DEFAULT"].withHeader().withDelimiter(';')>
+<#assign csvParser = CSVTool.parse(DataSources.get(0), cvsFormat)>
+<#assign users = DataFrameTool.toDataFrame(csvParser)>
+```
+
+### Select By Age
+
+```
+${DataFrameTool.print(users.select("(age > 40)"))}
+```
+
+which shows
+
+```
+┌────────────┬────────────┬────────────┐
+│#name │#age │#country │
+├────────────┼────────────┼────────────┤
+│Parker │45 │USA │
+├────────────┼────────────┼────────────┤
+│Peter │44 │Germany │
+└────────────┴────────────┴────────────┘
+```
+
+### Complex Select & Sort
+
+Now we want to create a new `DataFrame` by selecting `name` and `country`
+
+```
+<#assign country = "Germany">
+${DataFrameTool.print(users
+ .select("(name == 'Schmitt' || name == 'Meier') && country == '${country}'")
+ .sort("name", DataFrameTool.sortOrder["ASCENDING"]))}
+```
+
+which shows
+
+```
+┌────────────┬────────────┬────────────┐
+│#name │#age │#country │
+├────────────┼────────────┼────────────┤
+│Meier │20 │Germany │
+├────────────┼────────────┼────────────┤
+│Meier │24 │Germany │
+├────────────┼────────────┼────────────┤
+│Meier │30 │Germany │
+├────────────┼────────────┼────────────┤
+│Schmitt │24 │Germany │
+├────────────┼────────────┼────────────┤
+│Schmitt │30 │Germany │
+└────────────┴────────────┴────────────┘
+```
+
+### Count Column Values
+
+Let's assume we want to count the records for each `country`
+
+```
+${DataFrameTool.print(users.getColumn("country").transform(DataFrameTool.transformer["COUNT"]))}
+```
+
+returns the following `DataFrame`
+
+```
+┌────────────┬────────────┐
+│#country │#counts │
+├────────────┼────────────┤
+│Germany │6 │
+├────────────┼────────────┤
+│USA │1 │
+├────────────┼────────────┤
+│France │1 │
+├────────────┼────────────┤
+│UK │1 │
+└────────────┴────────────┘
+```
+
+### Group By Age And Country
+
+Let's assume that we want to group the `DataFrame` by `age` and `country`
+
+```
+${DataFrameTool.print(users.groupBy("age", "country").sort("age"))}
+```
+
+which results in
+
+```
+┌────────────┬────────────┐
+│#age │#country │
+├────────────┼────────────┤
+│20 │Germany │
+├────────────┼────────────┤
+│24 │Germany │
+├────────────┼────────────┤
+│30 │France │
+├────────────┼────────────┤
+│30 │Germany │
+├────────────┼────────────┤
+│33 │UK │
+├────────────┼────────────┤
+│44 │Germany │
+├────────────┼────────────┤
+│45 │USA │
+└────────────┴────────────┘
+```
+
+## JSON Examples
+
+Here we load a `site/samples/json/github-users.json` which represents a tabular data be
+being parsed as a list of maps and print the JSOB as dataframe
+
+```
+./bin/freemarker-cli \
+ -i '${DataFrameTool.print(DataFrameTool.fromMaps(GsonTool.parse(DataSources.get(0))))}' \
+ site/sample/json/github-users.json
+
+┌────────────┬────────────┬────────────┬────────────┬────────────┬────────────┬────────────┬────────────┬────────────┬────────────┬────────────┬────────────┬────────────┬────────────┬────────────┬────────────┬────────────┐
+│#login │#id │#avatar_ur │#gravatar_ │#url │#html_url │#followers │#following │#gists_url │#starred_u │#subscript │#organizat │#repos_url │#events_ur │#received_ │#type │#site_admi │
+├────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┤
+│mojombo │1.00000000 │https:/... │ │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │User │false │
+├────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┤
+│defunkt │2.00000000 │https:/... │ │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │User │true │
+├────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┤
+│pjhyett │3.00000000 │https:/... │ │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │User │true │
+├────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┤
+│wycats │4.00000000 │https:/... │ │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │User │false │
+├────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┤
+│ezmobius │5.00000000 │https:/... │ │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │User │false │
+├────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┤
+│ivey │6.00000000 │https:/... │ │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │User │false │
+├────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┤
+│evanphx │7.00000000 │https:/... │ │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │https:/... │User │false │
+└────────────┴────────────┴────────────┴────────────┴────────────┴────────────┴────────────┴────────────┴────────────┴────────────┴────────────┴────────────┴────────────┴────────────┴────────────┴────────────┴────────────┘
+```
+
+## Excel Examples
+
+Let's transform an Excel Sheet to a `DataFrame` being printed using the following template
+
+```
+<#assign dataSource = DataSources.get(0)>
+<#assign workbook = ExcelTool.parse(dataSource)>
+<#list ExcelTool.getSheets(workbook) as sheet>
+ <#assign table = ExcelTool.toTable(sheet)>
+ <#assign df = DataFrameTool.fromRows(table, true)>
+ ${DataFrameTool.print(df)}<#t>
+</#list>
+```
+
+which is rendered by the following command line invocation
+
+```
+./bin/freemarker-cli -t templates/excel/dataframe/transform.ftl site/sample/excel/test.xls
+
+┌────────────┬────────────┬────────────┬────────────┬────────────┬────────────┬────────────┐
+│#Text │#Date │#Number │#Currency │#Time │#Percentag │#Forumula │
+├────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┤
+│Row 1 │01/01/17 │100.00 │€100.00 │10:00 │50.00% │C2*F2 │
+├────────────┼────────────┼────────────┼────────────┼────────────┼────────────┼────────────┤
+│Row 2 │01/01/17 │100.00 │€100.00 │10:00 │50.00% │C3*F3 │
+└────────────┴────────────┴────────────┴────────────┴────────────┴────────────┴────────────┘
+```
\ No newline at end of file
diff --git a/freemarker-generator-cli/src/site/markdown/cli/usage/transforming-directories.md b/freemarker-generator-cli/src/site/markdown/cli/usage/transforming-directories.md
index 3181907..7a65dc0 100644
--- a/freemarker-generator-cli/src/site/markdown/cli/usage/transforming-directories.md
+++ b/freemarker-generator-cli/src/site/markdown/cli/usage/transforming-directories.md
@@ -5,16 +5,45 @@
* Transform an input directory recursively into an output directory
* If a template has a ".ftl" extension this extension will be removed after processing
* Only a single directory is support
-* Currently no inclusion / exclusion pattern for templates are supported
+* Currently no inclusion / exclusion patterns for templates are supported
+
+The following sample files are used
+
+* template/application.properties
+* template/nginx/nginx.conf.ftl
+
+```
+appassembler> tree site/template/
+site/template/
+|-- application.properties
+`-- nginx
+ `-- nginx.conf.ftl
+
+# == application.properties ==================================================
+server.name=${NGINX_HOSTNAME!"127.0.0.1"}
+server.logs=${NGINX_LOGS!"/var/log/nginx"}
+```
+
+```
+# == nginx-conf ==============================================================
+server {
+ listen ${NGINX_PORT!"80"};
+ server_name ${NGINX_HOSTNAME!"127.0.0.1"};
+
+ root ${NGINX_WEBROOT!"/usr/share/nginx/www"};
+ index index.htm;
+```
### Transform Template Directory To STDOUT
+If no output directory is provided all output is written to `stdout`
+
```
bin/freemarker-cli -t site/template/
# == application.properties ==================================================
server.name=localhost
server.logs=/var/log/nginx
-# == nginx-conf =============================================================
+# == nginx-conf ==============================================================
server {
listen 80;
server_name 127.0.0.1;
@@ -26,21 +55,30 @@
### Transform Template Directory To Output Directory
+The transformed templates are written to an `out` directory
+
+* `nginx.conf.ftl` was changed to `nginx.conf" during the transformation
+
```
-bin/freemarker-cli -t site/template/ -o out; ls -l out
-total 8
--rw-r--r-- 1 sgoeschl staff 128 May 30 20:02 application.properties
-drwxr-xr-x 3 sgoeschl staff 96 May 30 20:02 nginx
+bin/freemarker-cli -t site/template/ -o out; tree out
+out
+|-- application.properties
+`-- nginx
+ `-- nginx.conf
+
+1 directory, 2 files
```
-### Pass Parameter On The Command Line
+### Use Command Line Parameters
+
+A user-supplied parameter `NGINX_HOSTNAME` is used to render the templates
```
bin/freemarker-cli -t site/template/ -P NGINX_HOSTNAME=localhost
# == application.properties ==================================================
server.name=localhost
server.logs=/var/log/nginx
-# == nginx-conf =============================================================
+# == nginx-conf ==============================================================
server {
listen 80;
server_name localhost;
@@ -52,13 +90,18 @@
### Use Environment Variables
+All environment variables can be copied to the top-level data model by providing `-m env:///`
+
+* `-m` or `-data-model` creates a data model
+* `env:///` is an URI referencing all environment variables
+
```
export NGINX_PORT=8080
bin/freemarker-cli -t site/template/ -m env:///
# == application.properties ==================================================
server.name=localhost
server.logs=/var/log/nginx
-# == nginx-conf =============================================================
+# == nginx-conf ==============================================================
server {
listen 8080;
server_name 127.0.0.1;
@@ -70,13 +113,15 @@
### Use Environment File
+Instead of environment variables an environment file (aka properties file) can be used
+
```
echo "NGINX_PORT=8080" > nginx.env
bin/freemarker-cli -t site/template/ -m nginx.env
# == application.properties ==================================================
server.name=localhost
server.logs=/var/log/nginx
-# == nginx-conf =============================================================
+# == nginx-conf ==============================================================
server {
listen 8080;
server_name 127.0.0.1;
@@ -88,13 +133,15 @@
### Use JSON File
+Another option is passing the information as JSON file
+
```
echo '{"NGINX_PORT":"8443","NGINX_HOSTNAME":"localhost"}' > nginx.json
bin/freemarker-cli -t site/template/ -m nginx.json
# == application.properties ==================================================
server.name=localhost
server.logs=/var/log/nginx
-# == nginx-conf =============================================================
+# == nginx-conf ==============================================================
server {
listen 8443;
server_name localhost;
@@ -102,20 +149,42 @@
root /usr/share/nginx/www;
index index.htm;
}
+```
+### Use YAML File
+
+Yet another option is using a YAML file
+
+```
+echo -e "- NGINX_PORT": "\"8443\"\n- NGINX_HOSTNAME": "localhost" > nginx.yaml
+bin/freemarker-cli -t site/template/ -m nginx.yaml
+# == application.properties ==================================================
+server.name=localhost
+server.logs=/var/log/nginx
+# == nginx-conf ==============================================================
+server {
+ listen 8443;
+ server_name localhost;
+
+ root /usr/share/nginx/www;
+ index index.htm;
+}
```
### Use Environment Variable With JSON Payload
+In the cloud it is common to pass JSON configuration as environment variable
+
+* `env:///NGINX_CONF` selects the `NGINX_CONF` environment variable
+* `#mimetype=application/json` defines that JSON content is parsed
+
```
export NGINX_CONF='{"NGINX_PORT":"8443","NGINX_HOSTNAME":"localhost"}'
-echo $NGINX_CONF
-{"NGINX_PORT":"8443","NGINX_HOSTNAME":"localhost"}
bin/freemarker-cli -t site/template/ -m env:///NGINX_CONF#mimetype=application/json
# == application.properties ==================================================
server.name=localhost
server.logs=/var/log/nginx
-# == nginx-conf =============================================================
+# == nginx-conf ==============================================================
server {
listen 8443;
server_name localhost;
@@ -123,4 +192,26 @@
root /usr/share/nginx/www;
index index.htm;
}
-```
\ No newline at end of file
+```
+
+### Overriding Values From The Command Line
+
+For testing purpose it is useful to override certain settings
+
+```
+export NGINX_CONF='{"NGINX_PORT":"8443","NGINX_HOSTNAME":"localhost"}'
+bin/freemarker-cli -t site/template/ -PNGINX_HOSTNAME=www.mydomain.com -m env:///NGINX_CONF#mimetype=application/json
+# == application.properties ==================================================
+server.name=www.mydomain.com
+server.logs=/var/log/nginx
+# == nginx-conf ==============================================================
+server {
+ listen 8443;
+ server_name www.mydomain.com;
+
+ root /usr/share/nginx/www;
+ index index.htm;
+}
+```
+
+Please note that this only works for "top-level" variables, i.e. mimicking enviroment variables or property files.
\ No newline at end of file
diff --git a/freemarker-generator-cli/src/site/markdown/index.md b/freemarker-generator-cli/src/site/markdown/index.md
index b8d50e1..2eb0dfd 100644
--- a/freemarker-generator-cli/src/site/markdown/index.md
+++ b/freemarker-generator-cli/src/site/markdown/index.md
@@ -8,6 +8,10 @@
* [User-Supplied Parameters](cli/concepts/user-parameters.html)
* [Transformation](cli/concepts/transformation.html)
+### Tools
+
+* [DataFrameTool](cli/tools/dataframe.html)
+
### Usage
* [Transforming Directories](cli/usage/transforming-directories.html)
diff --git a/freemarker-generator-cli/src/test/data/json/list.json b/freemarker-generator-cli/src/test/data/json/list.json
new file mode 100644
index 0000000..40ea060
--- /dev/null
+++ b/freemarker-generator-cli/src/test/data/json/list.json
@@ -0,0 +1 @@
+["first", "second"]
\ No newline at end of file
diff --git a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
index 9b2727b..95c7241 100644
--- a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
+++ b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
@@ -62,6 +62,7 @@
assertValid(execute("-t templates/excel/md/transform.ftl site/sample/excel/test-multiple-sheets.xlsx"));
assertValid(execute("-t templates/excel/csv/transform.ftl site/sample/excel/test-multiple-sheets.xlsx"));
assertValid(execute("-t templates/excel/csv/custom.ftl -Pcsv.format=MYSQL site/sample/excel/test.xls"));
+ assertValid(execute("-t templates/excel/dataframe/transform.ftl site/sample/excel/test.xls"));
}
@Test
@@ -103,6 +104,11 @@
}
@Test
+ public void shouldRunDataFrameExamples() throws IOException {
+ assertValid(execute("-DCSV_TOOL_DELIMITER=SEMICOLON -DCSV_TOOL_HEADERS=true -t templates/dataframe/example.ftl site/sample/csv/dataframe.csv"));
+ }
+
+ @Test
public void shouldRunInteractiveTemplateExamples() throws IOException {
assertValid(execute("-i ${JsonPathTool.parse(DataSources.first).read(\"$.info.title\")} site/sample/json/swagger-spec.json"));
assertValid(execute("-i ${XmlTool.parse(DataSources.first)[\"recipients/person[1]/name\"]} site/sample/xml/recipients.xml"));
@@ -111,12 +117,13 @@
assertValid(execute("-i ${GsonTool.toJson(yaml)} -m yaml=site/sample/yaml/swagger-spec.yaml"));
assertValid(execute("-i ${YamlTool.toYaml(GsonTool.parse(DataSources.get(0)))} site/sample/json/swagger-spec.json"));
assertValid(execute("-i ${YamlTool.toYaml(json)} -m json=site/sample/json/swagger-spec.json"));
+ assertValid(execute("-i ${DataFrameTool.print(DataFrameTool.fromMaps(GsonTool.parse(DataSources.get(0))))} site/sample/json/github-users.json"));
}
@Test
public void shouldTransformTemplateDirectory() throws IOException {
- assertTrue(execute("-t site/template").contains("server.name=somehost"));
- assertTrue(execute("-t site/template -PNGINX_HOSTNAME=localhost").contains("server.name=localhost"));
+ assertTrue(execute("-t site/template").contains("server.name=127.0.0.1"));
+ assertTrue(execute("-t site/template -PNGINX_HOSTNAME=my.domain.com").contains("server.name=my.domain.com"));
}
@Test
diff --git a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
index 2dbaf2c..fd2c11c 100644
--- a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
+++ b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
@@ -48,8 +48,7 @@
// private static final String CMD = "-b ./src/test -t templates/demo.ftl -m env=./site/sample/properties/user_0001/user.properties";
// private static final String CMD = "-b ./src/test -t templates/demo.ftl -m ./site/sample/properties/user_0001/user.properties";
// private static final String CMD = "-b ./src/test --data-model post=https://jsonplaceholder.typicode.com/posts/2 -t templates/info.ftl";
- // private static final String CMD = "-b ./src/test -t templates/info.ftl -P name=value";
- private static final String CMD = "-P NGINX_PORT=8080 -t ../freemarker-generator-base/src/test/template -t templates/info.ftl";
+ private static final String CMD = "-DCSV_TOOL_DELIMITER=SEMICOLON -DCSV_TOOL_HEADERS=true -b ./src/test -t templates/dataframe/example.ftl https://raw.githubusercontent.com/nRo/DataFrame/master/src/test/resources/users.csv";
public static void main(String[] args) {
diff --git a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/config/DataModelSupplierTest.java b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/config/DataModelSupplierTest.java
index def6a86..1924ccc 100644
--- a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/config/DataModelSupplierTest.java
+++ b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/config/DataModelSupplierTest.java
@@ -20,6 +20,7 @@
import org.junit.Test;
import java.nio.file.Paths;
+import java.util.List;
import java.util.Map;
import static java.util.Collections.singletonList;
@@ -36,7 +37,7 @@
// === Environment Variables ===
@Test
- public void shouldResolveAllEnvironmentVariablesToTopLevelDataModel() {
+ public void shouldCopyAllEnvironmentVariablesToTopLevelDataModel() {
final DataModelSupplier supplier = supplier("env:///");
final Map<String, Object> model = supplier.get();
@@ -46,7 +47,7 @@
}
@Test
- public void shouldResolveAllEnvironmentVariablesToDataModelVariable() {
+ public void shouldCopyAllEnvironmentVariablesToDataModelVariable() {
final DataModelSupplier supplier = supplier("myenv=env:///");
final Map<String, Object> model = supplier.get();
@@ -57,7 +58,7 @@
}
@Test
- public void shouldResolveSingleEnvironmentVariablesToTopLevelDataModel() {
+ public void shouldCopySingleEnvironmentVariablesToTopLevelDataModel() {
final DataModelSupplier supplier = supplier("env:///PWD");
final Map<String, Object> model = supplier.get();
@@ -67,7 +68,7 @@
}
@Test
- public void shouldResolveSingleEnvironmentVariableToDataModelVariable() {
+ public void shouldCopySingleEnvironmentVariableToDataModelVariable() {
final DataModelSupplier supplier = supplier("mypwd=env:///PWD");
final Map<String, Object> model = supplier.get();
@@ -84,7 +85,7 @@
// === Properties ===
@Test
- public void shouldResolvePropertiesFileToTopLevelDataModel() {
+ public void shouldCopeyPropertiesFileToTopLevelDataModel() {
final DataModelSupplier supplier = supplier("./src/test/data/properties/test.properties");
final Map<String, Object> model = supplier.get();
@@ -95,7 +96,7 @@
}
@Test
- public void shouldResolvePropertiesFileToDataModelVariable() {
+ public void shouldCopyPropertiesFileToDataModelVariable() {
final DataModelSupplier supplier = supplier("props=./src/test/data/properties/test.properties");
final Map<String, Object> model = supplier.get();
@@ -106,7 +107,7 @@
}
@Test
- public void shouldResolvePropertiesUriToDataModelVariable() {
+ public void shouldCopyPropertiesUriToDataModelVariable() {
final DataModelSupplier supplier = supplier("props=file:///" + PWD + "/src/test/data/properties/test.properties");
final Map<String, Object> model = supplier.get();
@@ -119,7 +120,7 @@
// === JSON ===
@Test
- public void shouldResolveJsonFileToTopLevelDataModel() {
+ public void shouldCopyJsonObjectFileToTopLevelDataModel() {
final DataModelSupplier supplier = supplier("./src/test/data/json/environments.json");
final Map<String, Object> model = supplier.get();
@@ -130,6 +131,24 @@
}
@Test
+ public void shouldCopyJsonArrayFileToDataModelVariable() {
+ final DataModelSupplier supplier = supplier("list=./src/test/data/json/list.json");
+
+ final Map<String, Object> model = supplier.get();
+
+ assertEquals(1, model.size());
+ assertEquals("first", ((List)model.get("list")).get(0));
+ assertEquals("second", ((List)model.get("list")).get(1));
+ }
+
+ @Test(expected = Exception.class)
+ public void shouldFailWhenCopyJsonArrayFileToTopLevelDataModel() {
+ supplier("./src/test/data/json/list.json").get();
+ }
+
+ // == YAML ===
+
+ @Test
public void shouldResolveYamlFileToTopLevelDataModel() {
final DataModelSupplier supplier = supplier("./src/test/data/yaml/environments.yaml");
@@ -145,6 +164,16 @@
@Test
@Ignore
public void shouldResolveUrlToTopLevelDataModel() {
+ final DataModelSupplier supplier = supplier("https://jsonplaceholder.typicode.com/posts/2");
+
+ final Map<String, Object> model = supplier.get();
+
+ assertTrue(model.size() == 4);
+ }
+
+ @Test
+ @Ignore
+ public void shouldResolveUrlToDataModelVariable() {
final DataModelSupplier supplier = supplier("post=https://jsonplaceholder.typicode.com/posts/2");
final Map<String, Object> model = supplier.get();
@@ -155,16 +184,6 @@
@Test
@Ignore
- public void shouldResolveUrlToDataModelVariable() {
- final DataModelSupplier supplier = supplier("https://jsonplaceholder.typicode.com/posts/2");
-
- final Map<String, Object> model = supplier.get();
-
- assertTrue(model.size() == 4);
- }
-
- @Test(expected = RuntimeException.class)
- @Ignore
public void shouldResolveUrlToDataModelVariables() {
supplier("https://jsonplaceholder.typicode.com/posts/does-not-exist").get();
}
diff --git a/freemarker-generator-cli/src/test/templates/manual.ftl b/freemarker-generator-cli/src/test/templates/manual.ftl
index d33edf4..b381b05 100644
--- a/freemarker-generator-cli/src/test/templates/manual.ftl
+++ b/freemarker-generator-cli/src/test/templates/manual.ftl
@@ -17,10 +17,4 @@
-->
Manual Test
---------------------------------------------------------------------------
-<#assign smallNumber = 3.1415927>
-<#assign largeNumber = 99999.99>
-
-Small Number : ${smallNumber}
-Large Number : ${largeNumber}
-Date : ${.now?date}
-Time : ${.now?time}
+<#assign df=DataFrameTool.fromMaps(GsonTool.parse(DataSources.get(0)))>${DataFrameTool.print(df)}
\ No newline at end of file
diff --git a/freemarker-generator-cli/templates/dataframe/example.ftl b/freemarker-generator-cli/templates/dataframe/example.ftl
new file mode 100644
index 0000000..54d0ac2
--- /dev/null
+++ b/freemarker-generator-cli/templates/dataframe/example.ftl
@@ -0,0 +1,48 @@
+<#ftl output_format="plainText">
+<#--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<#assign csvParser = CSVTool.parse(DataSources.get(0))>
+<#assign users = DataFrameTool.fromCSVParser(csvParser)>
+
+Original Data
+=============================================================================
+${DataFrameTool.print(users)}
+
+Select By Age
+=============================================================================
+${DataFrameTool.print(users.select("(age > 40)"))}
+
+Select By Name & Country
+=============================================================================
+<#assign country = "Germany">
+${DataFrameTool.print(users
+.select("(name == 'Schmitt' || name == 'Meier') && country == '${country}'")
+.sort("name", DataFrameTool.sortOrder["ASCENDING"]))}
+
+Head of Users
+=============================================================================
+${DataFrameTool.print(users.head(2))}
+
+Count Column Values
+=============================================================================
+${DataFrameTool.print(users.getColumn("country").transform(DataFrameTool.transformer["COUNT"]))}
+
+Group By Age & Country
+=============================================================================
+${DataFrameTool.print(users.groupBy("country", "age").sort("country"))}
+
+
diff --git a/freemarker-generator-cli/templates/dataframe/html/print.ftl b/freemarker-generator-cli/templates/dataframe/html/print.ftl
new file mode 100644
index 0000000..e17d3a9
--- /dev/null
+++ b/freemarker-generator-cli/templates/dataframe/html/print.ftl
@@ -0,0 +1,54 @@
+<#ftl output_format="HTML" >
+<#--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<#assign cvsFormat = CSVTool.formats["DEFAULT"].withHeader().withDelimiter(';')>
+<#assign csvParser = CSVTool.parse(DataSources.get(0), cvsFormat)>
+<#assign dataFrame = DataFrameTool.toDataFrame(csvParser)>
+<#--------------------------------------------------------------------------->
+<!DOCTYPE html>
+<html>
+<head>
+ <title>DataFrame</title>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css">
+</head>
+<body>
+<div class="container-fluid">
+ <h1>DataFrame</h1>
+ <@writeDataFrame dataFrame/>
+</div>
+</body>
+</html>
+
+<#--------------------------------------------------------------------------->
+<#macro writeDataFrame dataFrame>
+ <table class="table table-striped">
+ <tr>
+ <#list dataFrame.columns as column>
+ <th>${column.name}</th>
+ </#list>
+ </tr>
+ <#list dataFrame.iterator() as row>
+ <tr>
+ <#list 0..row.size()-1 as idx>
+ <td>${row.getString(idx)}</td>
+ </#list>
+ </tr>
+ </#list>
+ </table>
+</#macro>
diff --git a/freemarker-generator-cli/templates/excel/dataframe/transform.ftl b/freemarker-generator-cli/templates/excel/dataframe/transform.ftl
new file mode 100644
index 0000000..d172bec
--- /dev/null
+++ b/freemarker-generator-cli/templates/excel/dataframe/transform.ftl
@@ -0,0 +1,23 @@
+<#--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<#assign dataSource = DataSources.get(0)>
+<#assign workbook = ExcelTool.parse(dataSource)>
+<#list ExcelTool.getSheets(workbook) as sheet>
+ <#assign table = ExcelTool.toTable(sheet)>
+ <#assign df = DataFrameTool.fromRows(table, true)>
+ ${DataFrameTool.print(df)}<#t>
+</#list>
diff --git a/freemarker-generator-tools/pom.xml b/freemarker-generator-tools/pom.xml
index 95e7d4c..57c6b59 100644
--- a/freemarker-generator-tools/pom.xml
+++ b/freemarker-generator-tools/pom.xml
@@ -54,6 +54,12 @@
<artifactId>commons-csv</artifactId>
<version>1.8</version>
</dependency>
+ <!-- DataFrame -->
+ <dependency>
+ <groupId>de.unknownreality</groupId>
+ <artifactId>dataframe</artifactId>
+ <version>0.7.6</version>
+ </dependency>
<!-- ExcelTool -->
<dependency>
<groupId>org.apache.poi</groupId>
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVTool.java
index 31c5d7c..296f903 100644
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVTool.java
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVTool.java
@@ -37,6 +37,7 @@
import java.util.function.Function;
import java.util.stream.Collectors;
+import static java.lang.Boolean.parseBoolean;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toList;
@@ -44,8 +45,10 @@
public class CommonsCSVTool {
+ private final CSVFormat defaulCSVFormat = csvFormat();
+
public CSVParser parse(DataSource dataSource) {
- return parse(dataSource, CSVFormat.DEFAULT);
+ return parse(dataSource, defaulCSVFormat);
}
public CSVParser parse(DataSource dataSource, CSVFormat format) {
@@ -65,7 +68,7 @@
}
public CSVParser parse(String csv) {
- return parse(csv, CSVFormat.DEFAULT);
+ return parse(csv, defaulCSVFormat);
}
public CSVParser parse(String csv, CSVFormat format) {
@@ -259,6 +262,23 @@
return result;
}
+ private CSVFormat csvFormat() {
+
+ CSVFormat csvFormat = CSVFormat.valueOf(System.getProperty("CSV_TOOL_FORMAT", "Default"));
+
+ final String delimiter = System.getProperty("CSV_TOOL_DELIMITER");
+ if (StringUtils.isNotEmpty(delimiter)) {
+ csvFormat = csvFormat.withDelimiter(toDelimiter(delimiter));
+ }
+
+ final boolean withHeader = parseBoolean(System.getProperty("CSV_TOOL_HEADERS", Boolean.toString(!csvFormat.getSkipHeaderRecord())));
+ if (withHeader) {
+ csvFormat = csvFormat.withHeader();
+ }
+
+ return csvFormat;
+ }
+
private static final class ValueResolver implements Function<CSVRecord, String> {
private final Integer index;
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java
new file mode 100644
index 0000000..5441861
--- /dev/null
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.freemarker.generator.tools.dataframe;
+
+import de.unknownreality.dataframe.DataFrame;
+import de.unknownreality.dataframe.DataFrameWriter;
+import de.unknownreality.dataframe.sort.SortColumn.Direction;
+import de.unknownreality.dataframe.transform.ColumnDataFrameTransform;
+import de.unknownreality.dataframe.transform.CountTransformer;
+import org.apache.commons.csv.CSVParser;
+import org.apache.freemarker.generator.tools.dataframe.converter.CSVConverter;
+import org.apache.freemarker.generator.tools.dataframe.converter.ListConverter;
+import org.apache.freemarker.generator.tools.dataframe.converter.MapConverter;
+
+import java.io.StringWriter;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static de.unknownreality.dataframe.DataFrameWriter.DEFAULT_PRINT_FORMAT;
+
+/**
+ * Create and manipulate data frames (tabular data structure). Data frames allow
+ * easy manipulation and transformation of data, e.g. joining two data frames.
+ * For more information see <a href="https://github.com/nRo/DataFrame">nRo/DataFrame</a>.
+ */
+public class DataFrameTool {
+
+ /**
+ * Create a data frame from Apache Commons CSVParser.
+ *
+ * @param csvParser CSV Parser
+ * @return data frame
+ */
+ public DataFrame fromCSVParser(CSVParser csvParser) {
+ return CSVConverter.toDataFrame(csvParser);
+ }
+
+ /**
+ * Create a data frame from a list of maps.
+ *
+ * @param maps maps to build the data frame
+ * @return data frame
+ */
+ public DataFrame fromMaps(Collection<Map<String, Object>> maps) {
+ return MapConverter.toDataFrame(maps);
+ }
+
+ /**
+ * Create a data frame from a list of rows.
+ *
+ * @param rows rows to build the data frame from
+ * @param withFirstRowAsColumnNames column names as first row?
+ * @return data frame
+ */
+ public DataFrame fromRows(List<List<Object>> rows, boolean withFirstRowAsColumnNames) {
+ return ListConverter.toDataFrame(rows, withFirstRowAsColumnNames);
+ }
+
+ /**
+ * Provide a convinience map with predefined sort orders to be used by templates.
+ *
+ * @return available sort orders
+ */
+ public Map<String, Direction> getSortOrder() {
+ final Map<String, Direction> result = new HashMap<>();
+ result.put(Direction.Ascending.name().toUpperCase(), Direction.Ascending);
+ result.put(Direction.Descending.name().toUpperCase(), Direction.Descending);
+ return result;
+ }
+
+ /**
+ * Provide a convinience map with predefined transformers.
+ *
+ * @return available transformers
+ */
+ public Map<String, ColumnDataFrameTransform> getTransformer() {
+ final Map<String, ColumnDataFrameTransform> result = new HashMap<>();
+ result.put("COUNT", countTransformer(false));
+ return result;
+ }
+
+ /**
+ * Print the <code>DataFrame</code> to the FreeMarker writer.
+ *
+ * @param dataFrame data frame
+ */
+ public String print(DataFrame dataFrame) {
+ final StringWriter writer = new StringWriter();
+ DataFrameWriter.write(writer, dataFrame, DEFAULT_PRINT_FORMAT);
+ return writer.toString();
+ }
+
+ @Override
+ public String toString() {
+ return "Bridge to nRo/DataFrame (see https://github.com/nRo/DataFrame)";
+ }
+
+ private static CountTransformer countTransformer(boolean ignoreNA) {
+ return new CountTransformer(ignoreNA);
+ }
+}
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/CSVConverter.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/CSVConverter.java
new file mode 100644
index 0000000..055b508
--- /dev/null
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/CSVConverter.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.freemarker.generator.tools.dataframe.converter;
+
+import de.unknownreality.dataframe.DataFrame;
+import de.unknownreality.dataframe.DataFrameBuilder;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+
+import java.io.IOException;
+import java.util.List;
+
+public class CSVConverter {
+
+ /**
+ * Create a data frame from Apache Commons CSV Parser.
+ *
+ * @param csvParser CSV Parser
+ * @return data frame
+ */
+ public static DataFrame toDataFrame(CSVParser csvParser) {
+ try {
+ final List<String> headerNames = csvParser.getHeaderNames();
+ final DataFrameBuilder builder = DataFrameBuilder.create();
+ final List<CSVRecord> records = csvParser.getRecords();
+ final CSVRecord firstRecord = records.get(0);
+
+ // build dataframe with headers
+ if (headerNames != null && !headerNames.isEmpty()) {
+ headerNames.forEach(builder::addStringColumn);
+ } else {
+ for (int i = 0; i < firstRecord.size(); i++) {
+ builder.addStringColumn(ConverterUtils.getAlphaColumnName(i + 1));
+ }
+ }
+
+ final DataFrame dataFrame = builder.build();
+
+ // populate rows
+ final String[] currValues = new String[firstRecord.size()];
+ for (CSVRecord csvRecord : records) {
+ for (int i = 0; i < currValues.length; i++) {
+ currValues[i] = csvRecord.get(i);
+ }
+ dataFrame.append(currValues);
+ }
+
+ return dataFrame;
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to create DataFrame", e);
+ }
+ }
+
+}
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ConverterUtils.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ConverterUtils.java
new file mode 100644
index 0000000..5237f52
--- /dev/null
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ConverterUtils.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.freemarker.generator.tools.dataframe.converter;
+
+import de.unknownreality.dataframe.DataFrame;
+import de.unknownreality.dataframe.DataFrameBuilder;
+import org.apache.freemarker.generator.base.table.Table;
+
+import java.util.List;
+
+public class ConverterUtils {
+
+ static DataFrame toDataFrame(Table table) {
+ final DataFrame dataFrame = create(table);
+ return appendValues(dataFrame, table);
+ }
+
+ static String getAlphaColumnName(int num) {
+ String result = "";
+ while (num > 0) {
+ num--; // 1 => a, not 0 => a
+ final int remainder = num % 26;
+ final char digit = (char) (remainder + 65);
+ result = digit + result;
+ num = (num - remainder) / 26;
+ }
+ return result;
+ }
+
+ private static DataFrameBuilder addColumn(DataFrameBuilder builder, String columnName, Class<?> columnType) {
+ switch (columnType.getName()) {
+ case "java.lang.Boolean":
+ return builder.addBooleanColumn(columnName);
+ case "java.lang.Byte":
+ return builder.addByteColumn(columnName);
+ case "java.lang.Double":
+ return builder.addDoubleColumn(columnName);
+ case "java.lang.Float":
+ return builder.addFloatColumn(columnName);
+ case "java.lang.Integer":
+ return builder.addIntegerColumn(columnName);
+ case "java.lang.Long":
+ return builder.addLongColumn(columnName);
+ case "java.lang.Short":
+ return builder.addShortColumn(columnName);
+ case "java.lang.String":
+ return builder.addStringColumn(columnName);
+ case "java.time.LocalDate":
+ return builder.addStringColumn(columnName);
+ case "java.time.LocalTime":
+ return builder.addStringColumn(columnName);
+ case "java.util.Date":
+ return builder.addStringColumn(columnName);
+ default:
+ throw new RuntimeException("Unable to add colum for the following type: " + columnType.getName());
+ }
+ }
+
+ private static Comparable<?>[] toComparables(List<?> values) {
+ final int size = values.size();
+ final Comparable<?>[] comparables = new Comparable<?>[size];
+ for (int i = 0; i < size; i++) {
+ comparables[i] = (Comparable<?>) values.get(i);
+ }
+ return comparables;
+ }
+
+ /**
+ * Create a <code>DataFrame</code> from a table.
+ *
+ * @param table table
+ * @return data frame
+ */
+ private static DataFrame create(Table table) {
+ final DataFrameBuilder builder = DataFrameBuilder.create();
+
+ if (table.hasColumnHeaderRow()) {
+ for (int i = 0; i < table.getColumnNames().size(); i++) {
+ final String columnName = table.getColumnNames().get(i);
+ final Class<?> columnType = table.getColumnTypes().get(i);
+ addColumn(builder, columnName, columnType);
+ }
+ } else {
+ if (!table.isEmpty()) {
+ final List<Object> firstRecord = table.getRow(0);
+ for (int i = 0; i < firstRecord.size(); i++) {
+ final String columnName = getAlphaColumnName(i + 1);
+ final Class<?> columnType = table.getColumnTypes().get(i);
+ addColumn(builder, columnName, columnType);
+ }
+ }
+ }
+
+ return builder.build();
+ }
+
+ private static DataFrame appendValues(DataFrame dataFrame, Table table) {
+ // TODO the conversion to Comparable[] is ugly
+ for (int i = 0; i < table.size(); i++) {
+ dataFrame.append(ConverterUtils.toComparables(table.getRow(i)));
+ }
+ return dataFrame;
+ }
+}
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ListConverter.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ListConverter.java
new file mode 100644
index 0000000..ecf891a
--- /dev/null
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ListConverter.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.freemarker.generator.tools.dataframe.converter;
+
+import de.unknownreality.dataframe.DataFrame;
+import org.apache.freemarker.generator.base.table.Table;
+
+import java.util.List;
+
+public class ListConverter {
+
+ /**
+ * Create a data frame from a list of rows. It is assumed
+ * that the rows represent tabular data.
+ *
+ * @param rows rows to build the data frame
+ * @return <code>DataFrame</code>
+ */
+ public static DataFrame toDataFrame(List<List<Object>> rows, boolean withFirstRowAsColumnNames) {
+ final Table table = Table.fromRows(rows, withFirstRowAsColumnNames);
+ return ConverterUtils.toDataFrame(table);
+ }
+}
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/MapConverter.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/MapConverter.java
new file mode 100644
index 0000000..f6800ce
--- /dev/null
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/MapConverter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.freemarker.generator.tools.dataframe.converter;
+
+import de.unknownreality.dataframe.DataFrame;
+import org.apache.freemarker.generator.base.table.Table;
+
+import java.util.Collection;
+import java.util.Map;
+
+import static java.util.Collections.singletonList;
+
+public class MapConverter {
+
+ /**
+ * Create a data frame from a list of maps. It is assumed
+ * that the map represent tabular data.
+ *
+ * @param map map to build the data frame
+ * @return <code>DataFrame</code>
+ */
+ public static DataFrame toDataFrame(Map<String, Object> map) {
+ return toDataFrame(singletonList(map));
+ }
+
+ /**
+ * Create a data frame from a list of maps. It is assumed
+ * that the map represent tabular data.
+ *
+ * @param maps list of map to build the data frame
+ * @return <code>DataFrame</code>
+ */
+ public static DataFrame toDataFrame(Collection<Map<String, Object>> maps) {
+ final Table table = Table.fromMaps(maps);
+ return ConverterUtils.toDataFrame(table);
+ }
+}
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/excel/ExcelTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/excel/ExcelTool.java
index f03df80..b5ead1d 100644
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/excel/ExcelTool.java
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/excel/ExcelTool.java
@@ -78,10 +78,10 @@
* @param sheet Excel sheet
* @return Table containing formatted cell values as strings
*/
- public List<List<String>> toTable(Sheet sheet) {
+ public List<List<Object>> toTable(Sheet sheet) {
final DataFormatter dataFormatter = dataFormatter();
final Iterator<Row> iterator = sheet.iterator();
- final List<List<String>> result = new ArrayList<>();
+ final List<List<Object>> result = new ArrayList<>();
while (iterator.hasNext()) {
final Row row = iterator.next();
@@ -117,8 +117,8 @@
return "Process Excels files (XLS, XLSX) using Apache POI (see https://poi.apache.org)";
}
- private static List<String> toColumns(Row row, DataFormatter dataFormatter) {
- final List<String> columnValues = new ArrayList<>();
+ private static List<Object> toColumns(Row row, DataFormatter dataFormatter) {
+ final List<Object> columnValues = new ArrayList<>();
for (int columnIndex = 0; columnIndex < row.getLastCellNum(); columnIndex++) {
final Cell cell = row.getCell(columnIndex, CREATE_NULL_AS_BLANK);
final String formatedCellValue = dataFormatter.formatCellValue(cell).trim();
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/gson/GsonTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/gson/GsonTool.java
index 6a864eb..dd8128b 100644
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/gson/GsonTool.java
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/gson/GsonTool.java
@@ -18,32 +18,49 @@
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
-import com.google.gson.reflect.TypeToken;
import com.google.gson.stream.JsonReader;
import org.apache.freemarker.generator.base.datasource.DataSource;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.lang.reflect.Type;
-import java.util.Map;
+/**
+ * JSON processing using <a href="https://github.com/google/gson">Google GSON</a>
+ */
public class GsonTool {
- private Gson gson;
- private Type type;
+ private volatile Gson gson;
- public Map<String, Object> parse(DataSource dataSource) {
+ /**
+ * Parse a data source containing a JSON object.
+ *
+ * @param dataSource data source
+ * @return parsed JSON either as a map or list
+ */
+ public Object parse(DataSource dataSource) {
try (JsonReader reader = new JsonReader(new InputStreamReader(dataSource.getUnsafeInputStream()))) {
- return gson().fromJson(reader, type());
+ return gson().fromJson(reader, Object.class);
} catch (IOException e) {
throw new RuntimeException("Failed to parse data source:" + dataSource, e);
}
}
- public Map<String, Object> parse(String json) {
- return gson().fromJson(json, type());
+ /**
+ * Parse a JSON object string.
+ *
+ * @param json Json string
+ * @return parsed JSON either as a map or list
+ */
+ public Object parse(String json) {
+ return gson().fromJson(json, Object.class);
}
+ /**
+ * Converts to JSON string.
+ *
+ * @param src source object
+ * @return JSON string
+ */
public String toJson(Object src) {
return gson().toJson(src);
}
@@ -59,11 +76,4 @@
}
return gson;
}
-
- private synchronized Type type() {
- if (type == null) {
- type = new TypeToken<Map<String, Object>>() {}.getType();
- }
- return type;
- }
}
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/jsoup/JsoupTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/jsoup/JsoupTool.java
index 6a9d23e..df252c3 100644
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/jsoup/JsoupTool.java
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/jsoup/JsoupTool.java
@@ -18,13 +18,14 @@
import org.apache.freemarker.generator.base.datasource.DataSource;
import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
import java.io.IOException;
import java.io.InputStream;
public class JsoupTool {
- public org.jsoup.nodes.Document parse(DataSource dataSource) {
+ public Document parse(DataSource dataSource) {
try (InputStream is = dataSource.getUnsafeInputStream()) {
return Jsoup.parse(is, dataSource.getCharset().name(), "");
} catch (IOException e) {
@@ -32,7 +33,7 @@
}
}
- public org.jsoup.nodes.Document parse(String html) {
+ public Document parse(String html) {
return Jsoup.parse(html);
}
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/snakeyaml/SnakeYamlTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/snakeyaml/SnakeYamlTool.java
index 1058d47..efacaea 100644
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/snakeyaml/SnakeYamlTool.java
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/snakeyaml/SnakeYamlTool.java
@@ -22,7 +22,6 @@
import java.io.IOException;
import java.io.InputStream;
-import java.util.Map;
import static org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK;
@@ -30,7 +29,7 @@
private Yaml yaml;
- public Map<String, Object> parse(DataSource dataSource) {
+ public Object parse(DataSource dataSource) {
try (InputStream is = dataSource.getUnsafeInputStream()) {
return yaml().load(is);
} catch (IOException e) {
@@ -38,8 +37,8 @@
}
}
- public Map<String, Object> parse(String value) {
- return yaml().load(value);
+ public Object parse(String yaml) {
+ return yaml().load(yaml);
}
public String toYaml(Object data) {
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/system/SystemTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/system/SystemTool.java
index 183b9da..ae5b66a 100644
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/system/SystemTool.java
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/system/SystemTool.java
@@ -34,7 +34,7 @@
import static org.apache.freemarker.generator.base.FreeMarkerConstants.Model.FREEMARKER_WRITER;
/**
- * Provides system related functionality, e.g. accessing environment variable,
+ * Provides system related functionality, e.g. accessing environment variables,
* system properties, commandl-line arguments, hostname, FreeMarker writer, etc.
*/
@SuppressWarnings("unchecked")
diff --git a/freemarker-generator-tools/src/test/data/csv/data_join_a.csv b/freemarker-generator-tools/src/test/data/csv/data_join_a.csv
new file mode 100644
index 0000000..cc05775
--- /dev/null
+++ b/freemarker-generator-tools/src/test/data/csv/data_join_a.csv
@@ -0,0 +1,5 @@
+GENE_ID;FPKM;CHR
+A;5;1
+B;4;2
+C;6;3
+D;6;1
\ No newline at end of file
diff --git a/freemarker-generator-tools/src/test/data/csv/data_join_b.csv b/freemarker-generator-tools/src/test/data/csv/data_join_b.csv
new file mode 100644
index 0000000..c84d1a0
--- /dev/null
+++ b/freemarker-generator-tools/src/test/data/csv/data_join_b.csv
@@ -0,0 +1,5 @@
+TRANSCRIPT_ID;GENE_ID;FPKM;TRANSCRIPT_NUMBER
+TA;A;7;1
+TB;A;3;2
+TC;B;6;1
+TD;E;4;1
\ No newline at end of file
diff --git a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVToolTest.java b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVToolTest.java
index f76e252..d8e4b6a 100644
--- a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVToolTest.java
+++ b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVToolTest.java
@@ -74,7 +74,6 @@
}
assertEquals(7, keys.size());
- assertEquals(7, keys.size());
assertEquals("C71", keys.get(0));
assertEquals("C72", keys.get(1));
assertEquals("C73", keys.get(2));
diff --git a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java
new file mode 100644
index 0000000..6965064
--- /dev/null
+++ b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.freemarker.generator.tools.dataframe;
+
+import de.unknownreality.dataframe.DataFrame;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.freemarker.generator.base.datasource.DataSourceFactory;
+import org.apache.freemarker.generator.tools.commonscsv.CommonsCSVTool;
+import org.apache.freemarker.generator.tools.excel.ExcelTool;
+import org.apache.freemarker.generator.tools.gson.GsonTool;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertNotNull;
+import static org.apache.commons.csv.CSVFormat.DEFAULT;
+
+public class DataFrameToolTest {
+
+ private static final String CSV_WITHOUT_HEADER = "A;5;1\n" +
+ "B;4;2\n" +
+ "C;6;3\n" +
+ "D;6;1";
+
+ private static final String CSV_WITH_HEADER = "GENE_ID;FPKM;CHR\n" +
+ CSV_WITHOUT_HEADER;
+
+ private static final String JSON_ARRAY = "[\n" +
+ " {\n" +
+ " \"Book ID\": \"1\",\n" +
+ " \"Book Name\": \"Computer Architecture\",\n" +
+ " \"Category\": \"Computers\",\n" +
+ " \"In Stock\": true,\n" +
+ " \"Price\": 125.60\n" +
+ " },\n" +
+ " {\n" +
+ " \"Book ID\": \"2\",\n" +
+ " \"Book Name\": \"Asp.Net 4 Blue Book\",\n" +
+ " \"Category\": \"Programming\",\n" +
+ " \"In Stock\": null,\n" +
+ " \"Price\": 56.00\n" +
+ " },\n" +
+ " {\n" +
+ " \"Book ID\": \"3\",\n" +
+ " \"Book Name\": \"Popular Science\",\n" +
+ " \"Category\": \"Science\",\n" +
+ " \"Price\": 210.40\n" +
+ " }\n" +
+ "]";
+
+ // === CSV ==============================================================
+
+ @Test
+ public void shouldParseCsvFileWithoutHeader() {
+ final CSVParser csvParser = csvParser(CSV_WITHOUT_HEADER, DEFAULT.withDelimiter(';'));
+ final DataFrame dataFrame = dataFrameTool().fromCSVParser(csvParser);
+
+ assertEquals(3, dataFrame.getColumns().size());
+ assertEquals(4, dataFrame.getRows().size());
+ assertEquals("A", dataFrame.getRow(0).get(0));
+ assertEquals("4", dataFrame.getRow(1).get(1));
+ assertEquals("3", dataFrame.getRow(2).get(2));
+ }
+
+ @Test
+ public void shouldParseCsvFileWithHeader() {
+ final CSVParser csvParser = csvParser(CSV_WITH_HEADER, DEFAULT.withHeader().withDelimiter(';'));
+ final DataFrame dataFrame = dataFrameTool().fromCSVParser(csvParser);
+
+ assertEquals(3, dataFrame.getColumns().size());
+ assertEquals(4, dataFrame.getRows().size());
+ assertEquals("A", dataFrame.getColumn("GENE_ID").get(0));
+ assertEquals("4", dataFrame.getColumn("FPKM").get(1));
+ assertEquals("3", dataFrame.getColumn("CHR").get(2));
+ }
+
+ // === JSON =============================================================
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void shouldParseJsonTable() {
+ final String columnName = "Book ID";
+ final List<Map<String, Object>> json = (List<Map<String, Object>>) gsonTool().parse(JSON_ARRAY);
+ final DataFrame dataFrame = dataFrameTool().fromMaps(json);
+
+ assertEquals(5, dataFrame.getColumns().size());
+ assertEquals(3, dataFrame.getRows().size());
+ assertEquals("1", dataFrame.getColumn(columnName).get(0));
+ assertEquals("2", dataFrame.getColumn(columnName).get(1));
+ assertEquals("3", dataFrame.getColumn(columnName).get(2));
+ }
+
+ // === Excel ============================================================
+
+ @Test
+ public void shouldParseExcelSheet() {
+ final ExcelTool excelTool = excelTool();
+ final Workbook workbook = excelTool.parse(DataSourceFactory.create("./src/test/data/excel/test.xls"));
+ final List<List<Object>> sheet = excelTool.toTable(workbook.getSheetAt(0));
+
+ final DataFrame dataFrame = dataFrameTool().fromRows(sheet, true);
+
+ assertEquals(7, dataFrame.getColumns().size());
+ assertEquals(2, dataFrame.getRows().size());
+ assertNotNull(dataFrame.getColumn("Text"));
+ assertNotNull(dataFrame.getColumn("Date"));
+ assertNotNull(dataFrame.getColumn("Number"));
+ assertNotNull(dataFrame.getColumn("Time"));
+ assertNotNull(dataFrame.getColumn("Percentage"));
+ assertNotNull(dataFrame.getColumn("Forumula"));
+ assertEquals("Row 1", dataFrame.getValue(0,0));
+ assertEquals("C3*F3", dataFrame.getColumn("Forumula").get(1));
+
+ }
+
+ private DataFrameTool dataFrameTool() {
+ return new DataFrameTool();
+ }
+
+ private CommonsCSVTool commonsCSVTool() {
+ return new CommonsCSVTool();
+ }
+
+ private GsonTool gsonTool() {
+ return new GsonTool();
+ }
+
+ private ExcelTool excelTool() {
+ return new ExcelTool();
+ }
+
+ private CSVParser csvParser(String csv, CSVFormat csvFormat) {
+ return commonsCSVTool().parse(csv, csvFormat);
+ }
+}
diff --git a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/excel/ExcelToolTest.java b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/excel/ExcelToolTest.java
index 9c8ef8b..089c2d0 100644
--- a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/excel/ExcelToolTest.java
+++ b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/excel/ExcelToolTest.java
@@ -39,7 +39,7 @@
final Workbook workbook = workbook(TEST_XLS);
final List<Sheet> sheets = excelTool().getSheets(workbook);
- final List<List<String>> records = excelTool().toTable(sheets.get(0));
+ final List<List<Object>> records = excelTool().toTable(sheets.get(0));
assertEquals(1, sheets.size());
assertEquals(3, records.size());
@@ -50,7 +50,7 @@
final Workbook workbook = workbook(TEST_XLSX);
final List<Sheet> sheets = excelTool().getSheets(workbook);
- final List<List<String>> records = excelTool().toTable(sheets.get(0));
+ final List<List<Object>> records = excelTool().toTable(sheets.get(0));
assertEquals(1, sheets.size());
assertEquals(3, records.size());
@@ -71,9 +71,9 @@
public void shouldConvertSheetToTable() {
final Workbook workbook = workbook(TEST_XLSX);
final List<Sheet> sheets = excelTool().getSheets(workbook);
- final List<List<String>> records = excelTool().toTable(sheets.get(0));
+ final List<List<Object>> records = excelTool().toTable(sheets.get(0));
- final List<String> record = records.get(1);
+ final List<Object> record = records.get(1);
assertEquals("Row 1", record.get(0));
assertEquals("01/31/17", record.get(1));
diff --git a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/gson/GsonToolTest.java b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/gson/GsonToolTest.java
index 7a5f5cf..601b2b4 100644
--- a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/gson/GsonToolTest.java
+++ b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/gson/GsonToolTest.java
@@ -29,13 +29,13 @@
public class GsonToolTest {
- private static final String JSON_OBJECT_STRING = "{\n" +
+ private static final String JSON_OBJECT = "{\n" +
" \"id\": 110.0,\n" +
" \"language\": \"Python\",\n" +
" \"price\": 1900.0\n" +
"}";
- private static final String JSON_ARRAY_STRING = "{\n" +
+ private static final String JSON_OBJECT_WITH_ARRAY = "{\n" +
" \"eBooks\": [\n" +
" {\n" +
" \"language\": \"Pascal\",\n" +
@@ -59,11 +59,33 @@
" \"color\": \"Red\"\n" +
"}";
+ private static final String JSON_ARRAY = "[\n" +
+ " {\n" +
+ " \"Book ID\": \"1\",\n" +
+ " \"Book Name\": \"Computer Architecture\",\n" +
+ " \"Category\": \"Computers\",\n" +
+ " \"Price\": \"125.60\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"Book ID\": \"2\",\n" +
+ " \"Book Name\": \"Asp.Net 4 Blue Book\",\n" +
+ " \"Category\": \"Programming\",\n" +
+ " \"Price\": \"56.00\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"Book ID\": \"3\",\n" +
+ " \"Book Name\": \"Popular Science\",\n" +
+ " \"Category\": \"Science\",\n" +
+ " \"Price\": \"210.40\"\n" +
+ " }\n" +
+ "]";
+
private final GsonTool gsonTool = gsonTool();
@Test
+ @SuppressWarnings("unchecked")
public void shouldParseJsonObject() {
- final Map<String, Object> map = parse(JSON_OBJECT_STRING);
+ final Map<String, Object> map = (Map) gsonTool.parse(JSON_OBJECT);
assertEquals(3, map.size());
assertEquals("110.0", map.get("id").toString());
@@ -72,39 +94,48 @@
}
@Test
- public void shouldParseJsonArray() {
- final Map<String, Object> map = parse(JSON_ARRAY_STRING);
+ @SuppressWarnings("unchecked")
+ public void shouldParseJsonObjectWithArray() {
+ final Map<String, Object> map = (Map) gsonTool.parse(JSON_OBJECT_WITH_ARRAY);
assertEquals(1, map.size());
assertEquals(3, ((List) map.get("eBooks")).size());
-
- return;
}
@Test
+ @SuppressWarnings("unchecked")
public void shouldParseJsonWithComemnts() {
- final Map<String, Object> map = parse(JSON_WITH_COMMENTS);
+ final Map<String, Object> map = (Map) gsonTool.parse(JSON_WITH_COMMENTS);
assertEquals("Apple", map.get("fruit"));
}
@Test
- public void shouldConvertToJson() {
- assertEquals(JSON_OBJECT_STRING, gsonTool.toJson(parse(JSON_OBJECT_STRING)));
- assertEquals(JSON_ARRAY_STRING, gsonTool.toJson(parse(JSON_ARRAY_STRING)));
+ @SuppressWarnings("unchecked")
+ public void shouldParseJsonArray() {
+ final List<Map<String, Object>> list = (List<Map<String, Object>>) gsonTool.parse(JSON_ARRAY);
+
+ assertEquals(3, list.size());
+ assertEquals("1", list.get(0).get("Book ID"));
+ assertEquals("2", list.get(1).get("Book ID"));
+ assertEquals("3", list.get(2).get("Book ID"));
}
@Test
- public void shouldParseComplexJson() throws IOException {
- final String json = readFileToString(new File("./src/test/data/json/swagger.json"), UTF_8);
- final Map<String, Object> map = parse(json);
-
- assertEquals("petstore.swagger.io", map.get("host"));
- assertEquals(json, gsonTool.toJson(parse(json)));
+ @SuppressWarnings("unchecked")
+ public void shouldConvertToJson() {
+ assertEquals(JSON_OBJECT, gsonTool.toJson(gsonTool.parse(JSON_OBJECT)));
+ assertEquals(JSON_OBJECT_WITH_ARRAY, gsonTool.toJson(gsonTool.parse(JSON_OBJECT_WITH_ARRAY)));
}
- private Map<String, Object> parse(String json) {
- return gsonTool.parse(json);
+ @Test
+ @SuppressWarnings("unchecked")
+ public void shouldParseComplexJson() throws IOException {
+ final String json = readFileToString(new File("./src/test/data/json/swagger.json"), UTF_8);
+ final Map<String, Object> map = (Map) gsonTool.parse(json);
+
+ assertEquals("petstore.swagger.io", map.get("host"));
+ assertEquals(json, gsonTool.toJson(gsonTool.parse(json)));
}
private GsonTool gsonTool() {
diff --git a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/snakeyaml/SnakeYamlToolTest.java b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/snakeyaml/SnakeYamlToolTest.java
index d588094..25ee6b1 100644
--- a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/snakeyaml/SnakeYamlToolTest.java
+++ b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/snakeyaml/SnakeYamlToolTest.java
@@ -34,16 +34,48 @@
private static final String ANY_GROUP = "group";
- private static final String ANY_YAML_STRING = "docker:\n" +
+ private static final String NESTED_YAML_MAP = "docker:\n" +
" - image: ubuntu:14.04\n" +
" - image: mongo:2.6.8\n" +
" command: [mongod, --smallfiles]\n" +
" - image: postgres:9.4.1";
+ private static final String MAP_YAML = "- NGINX_PORT: 8443\n" +
+ "- NGINX_HOSTNAME: localhost";
+
+ private static final String LIST_YAML = "- foo\n" +
+ "- bar";
+
+ @Test
+ public void shallParseSimpleListYamlString() {
+ final List<String> list = (List<String>) snakeYamlTool().parse(LIST_YAML);
+
+ assertEquals(2, list.size());
+ assertEquals("foo", list.get(0));
+ assertEquals("bar", list.get(1));
+ }
+
+ @Test
+ public void shallParseListOfMapYamlString() {
+ final List<Map<String, Object>> list = (List<Map<String, Object>>) snakeYamlTool().parse(MAP_YAML);
+
+ assertEquals(2, list.size());
+ assertEquals(8443, list.get(0).get("NGINX_PORT"));
+ assertEquals("localhost", list.get(1).get("NGINX_HOSTNAME"));
+ }
+
+ @Test
+ public void shallParseNestedYamlString() {
+ final Map<String, Object> map = (Map<String, Object>) snakeYamlTool().parse(NESTED_YAML_MAP);
+
+ assertEquals(1, map.size());
+ assertEquals(3, ((List<?>) map.get("docker")).size());
+ }
+
@Test
public void shallParseYamlDataSource() {
- try (DataSource dataSource = dataSource(ANY_YAML_STRING)) {
- final Map<String, Object> map = snakeYamlTool().parse(dataSource);
+ try (DataSource dataSource = dataSource(NESTED_YAML_MAP)) {
+ final Map<String, Object> map = (Map<String, Object>) snakeYamlTool().parse(dataSource);
assertEquals(1, map.size());
assertEquals(3, ((List<?>) map.get("docker")).size());
@@ -51,29 +83,21 @@
}
@Test
- public void shallParseYamlString() {
- final Map<String, Object> map = snakeYamlTool().parse(ANY_YAML_STRING);
+ public void shouldParseComplexYaml() throws IOException {
+ final String yaml = readFileToString(new File("./src/test/data/yaml/swagger.yaml"), UTF_8);
+ final Map<String, Object> map = (Map<String, Object>) snakeYamlTool().parse(yaml);
- assertEquals(1, map.size());
- assertEquals(3, ((List<?>) map.get("docker")).size());
+ assertEquals("2.0", map.get("swagger"));
+ assertEquals(16956, snakeYamlTool().toYaml(map).length());
}
@Test
public void shallConvertToYamlString() {
- final Map<String, Object> map = snakeYamlTool().parse(ANY_YAML_STRING);
+ final Map<String, Object> map = (Map<String, Object>) snakeYamlTool().parse(NESTED_YAML_MAP);
assertEquals(114, snakeYamlTool().toYaml(map).length());
}
- @Test
- public void shouldParseComplexYaml() throws IOException {
- final String yaml = readFileToString(new File("./src/test/data/yaml/swagger.yaml"), UTF_8);
- final Map<String, Object> map = snakeYamlTool().parse(yaml);
-
- assertEquals("2.0", map.get("swagger"));
- assertEquals(16956, snakeYamlTool().toYaml(map).length());
- }
-
private SnakeYamlTool snakeYamlTool() {
return new SnakeYamlTool();
}
diff --git a/travis.sh b/travis.sh
index e9c3f62..912d3d7 100755
--- a/travis.sh
+++ b/travis.sh
@@ -1,3 +1,4 @@
+#!/bin/sh
mvn clean install
cd ./freemarker-generator-cli
sh ./run-samples.sh
