Remove csv option from scan command #984 #1018
diff --git a/modules/cluster/src/main/java/org/apache/fluo/cluster/runner/AppRunner.java b/modules/cluster/src/main/java/org/apache/fluo/cluster/runner/AppRunner.java
index 5be9ef1..3ec090f 100644
--- a/modules/cluster/src/main/java/org/apache/fluo/cluster/runner/AppRunner.java
+++ b/modules/cluster/src/main/java/org/apache/fluo/cluster/runner/AppRunner.java
@@ -209,14 +209,6 @@
+ "internal schema, making it easier to comprehend.")
public boolean scanAccumuloTable = false;
- public boolean exportAsCsv = false;
- public String csvHeader;
- public String csvDelimiter;
- public String csvEscape;
- public String csvQuote;
- public String csvQuoteMode;
- public boolean exportAsJson = false;
-
public String getStartRow() {
return startRow;
}
@@ -242,8 +234,7 @@
public ScanUtil.ScanOpts getScanOpts() {
return new ScanUtil.ScanOpts(startRow, endRow, columns, exactRow, rowPrefix, help,
- hexEncNonAscii, scanAccumuloTable, exportAsCsv, csvDelimiter, csvEscape, csvHeader,
- csvQuote, csvQuoteMode, exportAsJson);
+ hexEncNonAscii, scanAccumuloTable, false);
}
}
}
diff --git a/modules/command/pom.xml b/modules/command/pom.xml
index f9949a2..8c8bf46 100644
--- a/modules/command/pom.xml
+++ b/modules/command/pom.xml
@@ -47,10 +47,6 @@
<artifactId>accumulo-core</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
</dependency>
diff --git a/modules/command/src/main/java/org/apache/fluo/command/FluoScan.java b/modules/command/src/main/java/org/apache/fluo/command/FluoScan.java
index 9f4455a..bcd03c0 100644
--- a/modules/command/src/main/java/org/apache/fluo/command/FluoScan.java
+++ b/modules/command/src/main/java/org/apache/fluo/command/FluoScan.java
@@ -4,9 +4,9 @@
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
@@ -20,7 +20,6 @@
import java.util.List;
import com.beust.jcommander.Parameter;
-import org.apache.commons.lang3.StringUtils;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.core.client.FluoAdminImpl;
import org.apache.fluo.core.util.ScanUtil;
@@ -56,31 +55,6 @@
+ "internal schema, making it easier to comprehend.")
public boolean scanAccumuloTable = false;
- @Parameter(names = "--csv", help = true,
- description = "Export key/values stored in Accumulo as CSV file. Uses Fluo application "
- + "properties to configure the CSV format.")
- public boolean exportAsCsv = false;
-
- @Parameter(names = "--csv-header", help = true, description = "Set header for \"true\".")
- public String csvHeader;
-
- @Parameter(names = "--csv-delimiter", help = true,
- description = "Configure delimiter to a designeted character.")
- public String csvDelimiter;
-
- @Parameter(names = "--csv-escape", help = true,
- description = "Configure escape to a designeted character.")
- public String csvEscape;
-
- @Parameter(names = "--csv-quote", help = true,
- description = "Configure quote to a designeted character.")
- public String csvQuote;
-
- @Parameter(names = "--csv-quote-mode", help = true,
- description = "Configure quote mode to a designeted mode. The possible "
- + "modes are: ALL, ALL_NON_NULL, MINIMAL, NONE and NON_NUMERIC")
- public String csvQuoteMode;
-
@Parameter(names = "--json", help = true,
description = "Export key/values stored in Accumulo as JSON file.")
public boolean exportAsJson = false;
@@ -101,26 +75,6 @@
return rowPrefix;
}
- public String getCsvHeader() {
- return csvHeader;
- }
-
- public String getCsvDelimiter() {
- return csvDelimiter;
- }
-
- public String getCsvEscape() {
- return csvEscape;
- }
-
- public String getCsvQuote() {
- return csvQuote;
- }
-
- public String getCsvQuoteMode() {
- return csvQuoteMode;
- }
-
public List<String> getColumns() {
if (columns == null) {
return Collections.emptyList();
@@ -130,25 +84,17 @@
/**
* Check if the parameters informed can be used together.
- * @since 1.2
*/
private void checkScanOptions() {
- if (this.exportAsCsv && this.exportAsJson) {
+ if (this.scanAccumuloTable && this.exportAsJson) {
throw new IllegalArgumentException(
- "Both \"--csv\" and \"--json\" can not be set together.");
- }
-
- if (!this.exportAsCsv && (StringUtils.isNotEmpty(this.csvDelimiter)
- | StringUtils.isNotEmpty(this.csvEscape) | StringUtils.isNotEmpty(this.csvHeader)
- | StringUtils.isNotEmpty(this.csvQuote) | StringUtils.isNotEmpty(this.csvQuoteMode))) {
- throw new IllegalArgumentException("No \"--csv\" detected");
+ "Both \"--raw\" and \"--json\" can not be set together.");
}
}
public ScanUtil.ScanOpts getScanOpts() {
return new ScanUtil.ScanOpts(startRow, endRow, columns, exactRow, rowPrefix, help,
- hexEncNonAscii, scanAccumuloTable, exportAsCsv, csvDelimiter, csvEscape, csvHeader,
- csvQuote, csvQuoteMode, exportAsJson);
+ hexEncNonAscii, scanAccumuloTable, exportAsJson);
}
public static ScanOptions parse(String[] args) {
diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index 0cfc6b3..fdd1f69 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -50,14 +50,6 @@
<artifactId>accumulo-core</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-csv</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
</dependency>
diff --git a/modules/core/src/main/java/org/apache/fluo/core/util/ScanUtil.java b/modules/core/src/main/java/org/apache/fluo/core/util/ScanUtil.java
index 78c3efb..4690fd5 100644
--- a/modules/core/src/main/java/org/apache/fluo/core/util/ScanUtil.java
+++ b/modules/core/src/main/java/org/apache/fluo/core/util/ScanUtil.java
@@ -22,24 +22,13 @@
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.function.Function;
-import com.google.common.collect.Iterables;
-import com.google.gson.FieldNamingPolicy;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonIOException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.commons.csv.CSVFormat;
-import org.apache.commons.csv.CSVPrinter;
-import org.apache.commons.csv.QuoteMode;
-import org.apache.commons.lang3.BooleanUtils;
-import org.apache.commons.lang3.ObjectUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.fluo.accumulo.format.FluoFormatter;
import org.apache.fluo.api.client.FluoClient;
import org.apache.fluo.api.client.FluoFactory;
@@ -50,14 +39,14 @@
import org.apache.fluo.api.data.Column;
import org.apache.fluo.api.data.RowColumnValue;
import org.apache.fluo.api.data.Span;
-import org.apache.fluo.api.exceptions.FluoException;
+
+import com.google.common.collect.Iterables;
+import com.google.gson.FieldNamingPolicy;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonIOException;
public class ScanUtil {
- public static final String CSV_HEADER = "csv.header";
- public static final String CSV_QUOTE_MODE = "csv.quoteMode";
- public static final String CSV_QUOTE = "csv.quote";
- public static final String CSV_ESCAPE = "csv.escape";
- public static final String CSV_DELIMITER = "csv.delimiter";
public static final String FLUO_VALUE = "value";
public static final String FLUO_COLUMN_VISIBILITY = "visibility";
public static final String FLUO_COLUMN_QUALIFIER = "qualifier";
@@ -115,6 +104,15 @@
return columns;
}
+
+ private static Function<Bytes, String> getEncoder(ScanOpts options) {
+ if (options.hexEncNonAscii) {
+ return Hex::encNonAscii;
+ } else {
+ return Bytes::toString;
+ }
+ }
+
public static void scanFluo(ScanOpts options, FluoConfiguration sConfig, PrintStream out)
throws IOException {
@@ -123,152 +121,56 @@
Span span = getSpan(options);
Collection<Column> columns = getColumns(options);
+ CellScanner cellScanner = s.scanner().over(span).fetch(columns).build();
+ Function<Bytes, String> encoder = getEncoder(options);
if (options.exportAsJson) {
- generateJson(options, span, columns, s, out);
- } else { // TSV or CSV format
- generateTsvCsv(options, span, columns, s, out);
- }
-
- } catch (FluoException e) {
- throw e;
- }
- }
- }
-
- /**
- * Generate TSV or CSV format as result of the scan.
- *
- * @since 1.2
- */
- private static void generateTsvCsv(ScanOpts options, Span span, Collection<Column> columns,
- final Snapshot snapshot, PrintStream out) throws IOException {
- // CSV Formater
- CSVFormat csvFormat = CSVFormat.DEFAULT;
- csvFormat = csvFormat.withQuoteMode(QuoteMode.ALL);
- csvFormat = csvFormat.withRecordSeparator("\n");
-
- // when "--csv" parameter is passed the "fluo.scan.csv" is analised
- if (options.exportAsCsv) {
- if (StringUtils.isNotEmpty(options.csvDelimiter)) {
- if (options.csvDelimiter.length() > 1) {
- throw new IllegalArgumentException(
- "Invalid character for the \"--csv-delimiter\" parameter.");
- }
- csvFormat = csvFormat.withDelimiter(options.csvDelimiter.charAt(0));
- }
-
- if (StringUtils.isNotEmpty(options.csvEscape)) {
- if (options.csvEscape.length() > 1) {
- throw new IllegalArgumentException(
- "Invalid character for the \"--csv-escape\" parameter.");
- }
- csvFormat = csvFormat.withEscape(options.csvEscape.charAt(0));
- }
-
- if (StringUtils.isNotEmpty(options.csvQuote)) {
- if (options.csvQuote.length() > 1) {
- throw new IllegalArgumentException(
- "Invalid character for the \"--csv-quote\" parameter.");
- }
- csvFormat = csvFormat.withQuote(options.csvQuote.charAt(0));
- }
-
- // It can throw "java.lang.IllegalArgumentException" if the value not exists
- // in "org.apache.commons.csv.QuoteMode"
- if (StringUtils.isNotEmpty(options.csvQuoteMode)) {
- csvFormat = csvFormat.withQuoteMode(QuoteMode.valueOf(options.csvQuoteMode));
- }
-
- if (BooleanUtils.toBooleanObject(
- ObjectUtils.defaultIfNull(options.csvHeader, Boolean.FALSE.toString()))) {
- csvFormat = csvFormat.withHeader(FLUO_ROW, FLUO_COLUMN_FAMILY, FLUO_COLUMN_QUALIFIER,
- FLUO_COLUMN_VISIBILITY, FLUO_VALUE);
- }
- } else {
- // Default TAB separator and NO quotes if possible.
- csvFormat = csvFormat.withDelimiter(CSVFormat.TDF.getDelimiter());
- csvFormat = csvFormat.withQuoteMode(QuoteMode.MINIMAL);
- }
-
- try (CSVPrinter printer = new CSVPrinter(out, csvFormat)) {
- CellScanner cellScanner = snapshot.scanner().over(span).fetch(columns).build();
-
- List<Object> record = new LinkedList<>();
- StringBuilder sb = new StringBuilder();
- int lines2check = 0;
- for (RowColumnValue rcv : cellScanner) {
- record.clear();
- if (options.hexEncNonAscii) {
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getRow());
- record.add(sb.toString());
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getColumn().getFamily());
- record.add(sb.toString());
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getColumn().getQualifier());
- record.add(sb.toString());
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getColumn().getVisibility());
- record.add(sb.toString());
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getValue());
- record.add(sb.toString());
+ generateJson(cellScanner, encoder, out);
} else {
- record.add(rcv.getsRow());
- record.add(rcv.getColumn().getFamily());
- record.add(rcv.getColumn().getQualifier());
- record.add(rcv.getColumn().getVisibility());
- record.add(rcv.getsValue());
- }
-
- printer.printRecord(record);
- lines2check++;
- if (lines2check == 100) {
- lines2check = 0;
- if (out.checkError()) {
- throw new IOException("Fail to write data to stream.");
+ for (RowColumnValue rcv : cellScanner) {
+ out.print(encoder.apply(rcv.getRow()));
+ out.print(' ');
+ out.print(encoder.apply(rcv.getColumn().getFamily()));
+ out.print(' ');
+ out.print(encoder.apply(rcv.getColumn().getQualifier()));
+ out.print(' ');
+ out.print(encoder.apply(rcv.getColumn().getVisibility()));
+ out.print("\t");
+ out.print(encoder.apply(rcv.getValue()));
+ out.println();
+ if (out.checkError()) {
+ break;
+ }
}
}
}
}
- out.flush();
}
/**
* Generate JSON format as result of the scan.
- *
+ *
* @since 1.2
*/
- private static void generateJson(ScanOpts options, Span span, Collection<Column> columns,
- final Snapshot snapshot, PrintStream out) throws JsonIOException {
+ private static void generateJson(CellScanner cellScanner, Function<Bytes, String> encoder,
+ PrintStream out) throws JsonIOException {
Gson gson = new GsonBuilder().serializeNulls().setDateFormat(DateFormat.LONG)
.setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES).setVersion(1.0)
.create();
- CellScanner cellScanner = snapshot.scanner().over(span).fetch(columns).build();
-
- StringBuilder sb = new StringBuilder();
Map<String, String> json = new LinkedHashMap<>();
for (RowColumnValue rcv : cellScanner) {
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getRow());
- json.put(FLUO_ROW, sb.toString());
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getColumn().getFamily());
- json.put(FLUO_COLUMN_FAMILY, sb.toString());
- Hex.encNonAscii(sb, rcv.getColumn().getQualifier());
- json.put(FLUO_COLUMN_QUALIFIER, sb.toString());
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getColumn().getVisibility());
- json.put(FLUO_COLUMN_VISIBILITY, sb.toString());
- sb.setLength(0);
- Hex.encNonAscii(sb, rcv.getValue());
- json.put(FLUO_VALUE, sb.toString());
-
+ json.put(FLUO_ROW, encoder.apply(rcv.getRow()));
+ json.put(FLUO_COLUMN_FAMILY, encoder.apply(rcv.getColumn().getFamily()));
+ json.put(FLUO_COLUMN_QUALIFIER, encoder.apply(rcv.getColumn().getQualifier()));
+ json.put(FLUO_COLUMN_VISIBILITY, encoder.apply(rcv.getColumn().getVisibility()));
+ json.put(FLUO_VALUE, encoder.apply(rcv.getValue()));
gson.toJson(json, out);
out.append("\n");
+
+ if (out.checkError()) {
+ break;
+ }
}
out.flush();
}
@@ -311,18 +213,11 @@
public boolean help;
public boolean hexEncNonAscii = true;
public boolean scanAccumuloTable = false;
- public boolean exportAsCsv = false;
public boolean exportAsJson = false;
- public final String csvDelimiter;
- public final String csvEscape;
- public final String csvHeader;
- public final String csvQuote;
- public final String csvQuoteMode;
public ScanOpts(String startRow, String endRow, List<String> columns, String exactRow,
String rowPrefix, boolean help, boolean hexEncNonAscii, boolean scanAccumuloTable,
- boolean exportAsCsv, String csvDelimiter, String csvEscape, String csvHeader,
- String csvQuote, String csvQuoteMode, boolean exportAsJson) {
+ boolean exportAsJson) {
this.startRow = startRow;
this.endRow = endRow;
this.columns = columns;
@@ -331,12 +226,6 @@
this.help = help;
this.hexEncNonAscii = hexEncNonAscii;
this.scanAccumuloTable = scanAccumuloTable;
- this.exportAsCsv = exportAsCsv;
- this.csvDelimiter = csvDelimiter;
- this.csvEscape = csvEscape;
- this.csvHeader = csvHeader;
- this.csvQuote = csvQuote;
- this.csvQuoteMode = csvQuoteMode;
this.exportAsJson = exportAsJson;
}
diff --git a/modules/distribution/src/main/lib/fetch.sh b/modules/distribution/src/main/lib/fetch.sh
index 51a54f4..2dd523c 100755
--- a/modules/distribution/src/main/lib/fetch.sh
+++ b/modules/distribution/src/main/lib/fetch.sh
@@ -62,8 +62,6 @@
download commons-collections:commons-collections:jar:3.2.1
download commons-configuration:commons-configuration:jar:1.10
download commons-io:commons-io:jar:2.4
- download org.apache.commons:commons-csv:jar:1.5
- download org.apache.commons:commons-lang3:jar:3.7
download io.dropwizard.metrics:metrics-core:jar:3.1.1
download io.dropwizard.metrics:metrics-graphite:jar:3.1.1
download javax.inject:javax.inject:jar:1
diff --git a/pom.xml b/pom.xml
index 7307daa..c4f8f3e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -140,16 +140,6 @@
<version>${accumulo.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-csv</artifactId>
- <version>1.5</version>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <version>3.7</version>
- </dependency>
- <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
<version>${curator.version}</version>
