NIFI-8273: Fix typos in docs/clarify some statements. Addressed handling of arrays as partitions
This closes #4948.
diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedPartitionRecord.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedPartitionRecord.java
index d0cb89f..2890436 100644
--- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedPartitionRecord.java
+++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedPartitionRecord.java
@@ -50,6 +50,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -67,7 +68,8 @@
explanation = "Provides operator the ability to execute arbitrary code assuming all permissions that NiFi has.")
})
@WritesAttributes({
- @WritesAttribute(attribute = "partition", description = "The partition of the outgoing flow file."),
+ @WritesAttribute(attribute = "partition", description = "The partition of the outgoing flow file. If the script indicates that the partition has a null value, the attribute will be set to " +
+ "the literal string \"<null partition>\" (without quotes). Otherwise, the attribute is set to the String representation of whatever value is returned by the script."),
@WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Record Writer"),
@WritesAttribute(attribute = "record.count", description = "The number of records within the flow file."),
@WritesAttribute(attribute = "record.error.message", description = "This attribute provides on failure the error message encountered by the Reader or Writer."),
@@ -167,8 +169,8 @@
final RecordSet recordSet = reader.createRecordSet();
final PushBackRecordSet pushBackSet = new PushBackRecordSet(recordSet);
- final Map<String, FlowFile> outgoingFlowFiles = new HashMap<>();
- final Map<String, RecordSetWriter> recordSetWriters = new HashMap<>();
+ final Map<Object, FlowFile> outgoingFlowFiles = new HashMap<>();
+ final Map<Object, RecordSetWriter> recordSetWriters = new HashMap<>();
// Reading in records and evaluate script
while (pushBackSet.isAnotherRecord()) {
@@ -177,7 +179,7 @@
getLogger().debug("Evaluated scripted against {} (index {}), producing result of {}", record, counts.getRecordCount(), evaluatedValue);
counts.incrementRecordCount();
- final String partition = (evaluatedValue == null) ? null : evaluatedValue.toString();
+ final Object partition = (evaluatedValue instanceof Object[]) ? Arrays.asList((Object[]) evaluatedValue) : evaluatedValue;
RecordSetWriter writer = recordSetWriters.get(partition);
if (writer == null) {
@@ -196,13 +198,13 @@
// Sending outgoing flow files
int fragmentIndex = 0;
- for (final String partition : outgoingFlowFiles.keySet()) {
+ for (final Object partition : outgoingFlowFiles.keySet()) {
final RecordSetWriter writer = recordSetWriters.get(partition);
final FlowFile outgoingFlowFile = outgoingFlowFiles.get(partition);
final Map<String, String> attributes = new HashMap<>(incomingFlowFile.getAttributes());
attributes.put("mime.type", writer.getMimeType());
- attributes.put("partition", partition);
+ attributes.put("partition", partition == null ? "<null partition>" : partition.toString());
attributes.put("fragment.index", String.valueOf(fragmentIndex));
attributes.put("fragment.count", String.valueOf(outgoingFlowFiles.size()));
diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/resources/docs/org.apache.nifi.processors.script.ScriptedPartitonRecord/additionalDetails.html b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/resources/docs/org.apache.nifi.processors.script.ScriptedPartitonRecord/additionalDetails.html
index 585088a..e68a786 100644
--- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/resources/docs/org.apache.nifi.processors.script.ScriptedPartitonRecord/additionalDetails.html
+++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/resources/docs/org.apache.nifi.processors.script.ScriptedPartitonRecord/additionalDetails.html
@@ -33,14 +33,14 @@
<h3>Description</h3>
<p>
- The ScriptedPartitionRecord provides the ability to use a scripting language, such as Groovy or Jython, to quickly and easily partition a Record based on the contents of it.
+ The ScriptedPartitionRecord provides the ability to use a scripting language, such as Groovy or Jython, to quickly and easily partition a Record based on its contents.
There are multiple ways to reach the same behaviour such as using PartitionRecord but working with user provided scripts opens a wide range of possibilities on the decision
logic of partitioning the individual records.
</p>
<p>
The provided script is evaluated once for each Record that is encountered in the incoming FlowFile. Each time that the script is invoked, it is expected to return an <code>object</code> or a <code>null</code> value.
- The string representation of the return value is used as "partition". The <code>null</code> value is handled separately without conversion into string.
+ The string representation of the return value is used as the record's "partition". The <code>null</code> value is handled separately without conversion into string.
All Records with the same partition then will be batched to one FlowFile and routed to the <code>success</code> Relationship.
</p>
@@ -89,17 +89,14 @@
<h3>Return Value</h3>
<p>
- The script is invoked separately for each Record. It is accepted to return with any Object values might be represented as string. This string value will be used as the partition of the given Record.
- Alternately it is possible to return with <code>null</code> value.
-
- Each time the script is invoked, it is expected to return a <code>boolean</code> value. Return values other than <code>boolean</code>, including <code>null</code> value will be handled as
- unexpected script behaviour and handled accordingly: the processing will be interrupted and the incoming FlowFile will be transferred to the <code>failure</code> relationship without further execution.
+ The script is invoked separately for each Record. It is acceptable to return any Object might be represented as string. This string value will be used as the partition of the given
+ Record. Additionally, the script may return <code>null</code>.
</p>
<h2>Example</h2>
<p>
- The following script
+ The following script will partition the input on the value of the "stellarType" field.
</p>
<p>
diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/resources/docs/org.apache.nifi.processors.script.ScriptedValidateRecord/additionalDetails.html b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/resources/docs/org.apache.nifi.processors.script.ScriptedValidateRecord/additionalDetails.html
index 965b068..3794156 100644
--- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/resources/docs/org.apache.nifi.processors.script.ScriptedValidateRecord/additionalDetails.html
+++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/resources/docs/org.apache.nifi.processors.script.ScriptedValidateRecord/additionalDetails.html
@@ -34,8 +34,9 @@
<p>
The ScriptedValidateRecord Processor provides the ability to use a scripting language, such as Groovy or Jyton in order to validate Records in an incoming FlowFile.
- NiFi provides several different Processors that can be used to work with Records in different ways. Each of these processors has its pros and cons.
- The ScriptedValidateRecord is intended to work together with these processors and be used as a preliminary step in order to prevent more complex processing steps from working with incorrect Records.
+ The provided script will be evaluated against each Record in an incoming FlowFile. Each of those records will then be routed to either the "valid" or "invalid" FlowFile.
+ As a result, each incoming FlowFile may be broken up into two individual FlowFiles (if some records are valid and some are invalid, according to the script), or the incoming FlowFile
+ may have all of its Records kept together in a single FlowFile, routed to either "valid" or "invalid" (if all records are valid or if all records are invalid, according to the script).
</p>
<p>
diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestScriptedPartitionRecord.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestScriptedPartitionRecord.java
index 1c169cd..c8d4178 100644
--- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestScriptedPartitionRecord.java
+++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestScriptedPartitionRecord.java
@@ -37,7 +37,7 @@
private static final String PARTITION_1 = "partition1";
private static final String PARTITION_2 = "partition2";
private static final Integer PARTITION_3 = 3;
- private static final String PARTITION_4 = null;
+ private static final String PARTITION_4 = "<null partition>";
@Test
public void testIncomingFlowFileContainsNoRecords() {
