Merge pull request #1718 from mattcasters/master
HOP-4270
diff --git a/integration-tests/beam_directrunner/0008-target-output-transform-validation.hpl b/integration-tests/beam_directrunner/0008-target-output-transform-validation.hpl
new file mode 100644
index 0000000..e0ba29b
--- /dev/null
+++ b/integration-tests/beam_directrunner/0008-target-output-transform-validation.hpl
@@ -0,0 +1,301 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0008-target-output-transform-validation</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2022/10/04 14:46:47.501</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2022/10/04 14:46:47.501</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>/tmp/0008/target-output*.csv</from>
+ <to>Validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>/tmp/0008/target-output*.csv</name>
+ <type>TextFileInput2</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <accept_filenames>N</accept_filenames>
+ <passing_through_fields>N</passing_through_fields>
+ <accept_field/>
+ <accept_transform_name/>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <enclosure_breaks>N</enclosure_breaks>
+ <escapechar/>
+ <header>N</header>
+ <nr_headerlines>1</nr_headerlines>
+ <footer>N</footer>
+ <nr_footerlines>1</nr_footerlines>
+ <line_wrapped>N</line_wrapped>
+ <nr_wraps>1</nr_wraps>
+ <layout_paged>N</layout_paged>
+ <nr_lines_per_page>80</nr_lines_per_page>
+ <nr_lines_doc_header>0</nr_lines_doc_header>
+ <noempty>Y</noempty>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <rownumByFile>N</rownumByFile>
+ <rownum_field/>
+ <format>Unix</format>
+ <encoding/>
+ <length>Characters</length>
+ <add_to_result_filenames>Y</add_to_result_filenames>
+ <file>
+ <name>${java.io.tmpdir}/0008/</name>
+ <filemask>.*\.csv$</filemask>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ <type>CSV</type>
+ <compression>None</compression>
+ </file>
+ <filters>
+ </filters>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <format> #</format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>15</position>
+ <length>0</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>Last name</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>15</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>First name</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>20</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>cust_zip_code</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>15</position>
+ <length>0</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>city</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>8</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>birthdate</name>
+ <type>Date</type>
+ <format>yyyy/MM/dd</format>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>-1</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>street</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>11</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>housenr</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>15</position>
+ <length>0</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>stateCode</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>9</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>state</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <ifnull/>
+ <position>30</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ </fields>
+ <limit>0</limit>
+ <error_ignored>N</error_ignored>
+ <skip_bad_files>N</skip_bad_files>
+ <file_error_field/>
+ <file_error_message_field/>
+ <error_line_skipped>N</error_line_skipped>
+ <error_count_field/>
+ <error_fields_field/>
+ <error_text_field/>
+ <bad_line_files_destination_directory/>
+ <bad_line_files_extension>warning</bad_line_files_extension>
+ <error_line_files_destination_directory/>
+ <error_line_files_extension>error</error_line_files_extension>
+ <line_number_files_destination_directory/>
+ <line_number_files_extension>line</line_number_files_extension>
+ <date_format_lenient>Y</date_format_lenient>
+ <date_format_locale>en_US</date_format_locale>
+ <shortFileFieldName/>
+ <pathFieldName/>
+ <hiddenFieldName/>
+ <lastModificationTimeFieldName/>
+ <uriNameFieldName/>
+ <rootUriNameFieldName/>
+ <extensionFieldName/>
+ <sizeFieldName/>
+ <attributes/>
+ <GUI>
+ <xloc>176</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Validate</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>464</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/beam_directrunner/0008-target-output-transform.hpl b/integration-tests/beam_directrunner/0008-target-output-transform.hpl
new file mode 100644
index 0000000..baae6f6
--- /dev/null
+++ b/integration-tests/beam_directrunner/0008-target-output-transform.hpl
@@ -0,0 +1,133 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0008-target-output-transform</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2022/10/04 14:43:21.769</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2022/10/04 14:43:21.769</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>input/customers-noheader-1k.txt</from>
+ <to>CA Only</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>CA Only</from>
+ <to>/tmp/0008/target-output.csv</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>/tmp/0008/target-output.csv</name>
+ <type>BeamOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <file_prefix>target-output</file_prefix>
+ <file_suffix>.csv</file_suffix>
+ <output_location>${java.io.tmpdir}/0008/</output_location>
+ <windowed>N</windowed>
+ <attributes/>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>CA Only</name>
+ <type>FilterRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <send_true_to>/tmp/0008/target-output.csv</send_true_to>
+ <send_false_to/>
+ <compare>
+ <condition>
+ <negated>N</negated>
+ <leftvalue>stateCode</leftvalue>
+ <function>=</function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>String</type>
+ <text>CA</text>
+ <length>-1</length>
+ <precision>-1</precision>
+ <isnull>N</isnull>
+ <mask/>
+ </value>
+ </condition>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>384</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>input/customers-noheader-1k.txt</name>
+ <type>BeamInput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <file_description_name>Customers</file_description_name>
+ <input_location>${PROJECT_HOME}/input/customers-noheader-1k.txt</input_location>
+ <attributes/>
+ <GUI>
+ <xloc>176</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/beam_directrunner/datasets/0008-target-output.csv b/integration-tests/beam_directrunner/datasets/0008-target-output.csv
new file mode 100644
index 0000000..67e45ca
--- /dev/null
+++ b/integration-tests/beam_directrunner/datasets/0008-target-output.csv
@@ -0,0 +1,20 @@
+id,Last name,First name,cust_zip_code,city,birthdate,street,housenr,stateCode,state
+" 12",vnaov-name,wha-firstname," 13120",egm-city,1954/03/28,hpep-street," 20",CA,CALIFORNIA
+" 807",pvcck-name,nnx-firstname," 19780",kfm-city,1962/07/05,aybu-street," 151",CA,CALIFORNIA
+" 308",ibfdt-name,sfu-firstname," 11120",kqf-city,1954/06/22,xbwg-street," 132",CA,CALIFORNIA
+" 731",yprgp-name,vgc-firstname," 14660",alm-city,1971/01/05,uoxp-street," 110",CA,CALIFORNIA
+" 181",bxerk-name,kxv-firstname," 14180",sek-city,1982/02/18,ctwu-street," 84",CA,CALIFORNIA
+" 44",wzkjq-name,rgh-firstname," 19000",hkm-city,1974/08/12,yixf-street," 134",CA,CALIFORNIA
+" 756",jdmdi-name,llu-firstname," 18060",pss-city,1974/07/01,utrs-street," 198",CA,CALIFORNIA
+" 304",cqapx-name,skq-firstname," 11080",akw-city,1958/09/02,xakx-street," 14",CA,CALIFORNIA
+" 919",papgo-name,oyu-firstname," 10280",bet-city,1970/06/19,otzt-street," 155",CA,CALIFORNIA
+" 600",mdftt-name,ldq-firstname," 11460",mkm-city,1966/09/03,rjkt-street," 9",CA,CALIFORNIA
+" 177",nltvw-name,ahc-firstname," 19520",uxf-city,1958/03/16,fwsy-street," 131",CA,CALIFORNIA
+" 26",rezku-name,zio-firstname," 19080",nvt-city,1982/07/14,wwkd-street," 91",CA,CALIFORNIA
+" 272",qxjcn-name,wpy-firstname," 11260",rew-city,1954/12/19,tldv-street," 115",CA,CALIFORNIA
+" 422",fmrwo-name,wlf-firstname," 17820",qwb-city,1962/05/03,stcz-street," 22",CA,CALIFORNIA
+" 912",mpkrp-name,etl-firstname," 10560",kty-city,1958/06/02,cpug-street," 67",CA,CALIFORNIA
+" 561",avxuw-name,niq-firstname," 13520",cyx-city,1958/05/29,dxit-street," 167",CA,CALIFORNIA
+" 135",eqley-name,ttv-firstname," 12740",trs-city,1974/09/16,ibff-street," 187",CA,CALIFORNIA
+" 997",nfdtl-name,jxu-firstname," 12980",isx-city,1982/07/05,dldc-street," 166",CA,CALIFORNIA
+" 25",bwkoe-name,ayl-firstname," 18660",rtw-city,1978/07/16,mzww-street," 179",CA,CALIFORNIA
diff --git a/integration-tests/beam_directrunner/main-0008-target-output-transform.hwf b/integration-tests/beam_directrunner/main-0008-target-output-transform.hwf
new file mode 100644
index 0000000..fcf1dc7
--- /dev/null
+++ b/integration-tests/beam_directrunner/main-0008-target-output-transform.hwf
@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0008-target-output-transform</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2022/10/04 14:43:03.589</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2022/10/04 14:43:03.589</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>96</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0008-target-output-transform</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0008-target-output-transform.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run Pipeline Unit Tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0008-target-output-transform-validation UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>512</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0008-target-output-transform</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>0008-target-output-transform</from>
+ <to>Run Pipeline Unit Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/beam_directrunner/metadata/dataset/0008-target-output.json b/integration-tests/beam_directrunner/metadata/dataset/0008-target-output.json
new file mode 100644
index 0000000..f19a5d1
--- /dev/null
+++ b/integration-tests/beam_directrunner/metadata/dataset/0008-target-output.json
@@ -0,0 +1,88 @@
+{
+ "base_filename": "0008-target-output.csv",
+ "name": "0008-target-output",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": 0,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_format": " #",
+ "field_name": "id"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "Last name"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "First name"
+ },
+ {
+ "field_comment": "",
+ "field_length": 0,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "cust_zip_code"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "city"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 3,
+ "field_precision": -1,
+ "field_format": "yyyy/MM/dd",
+ "field_name": "birthdate"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "street"
+ },
+ {
+ "field_comment": "",
+ "field_length": 0,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "housenr"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "stateCode"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "state"
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git a/integration-tests/beam_directrunner/metadata/unit-test/0008-target-output-transform-validation UNIT.json b/integration-tests/beam_directrunner/metadata/unit-test/0008-target-output-transform-validation UNIT.json
new file mode 100644
index 0000000..291935f
--- /dev/null
+++ b/integration-tests/beam_directrunner/metadata/unit-test/0008-target-output-transform-validation UNIT.json
@@ -0,0 +1,64 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": true,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "Last name",
+ "data_set_field": "Last name"
+ },
+ {
+ "transform_field": "First name",
+ "data_set_field": "First name"
+ },
+ {
+ "transform_field": "cust_zip_code",
+ "data_set_field": "cust_zip_code"
+ },
+ {
+ "transform_field": "city",
+ "data_set_field": "city"
+ },
+ {
+ "transform_field": "birthdate",
+ "data_set_field": "birthdate"
+ },
+ {
+ "transform_field": "street",
+ "data_set_field": "street"
+ },
+ {
+ "transform_field": "housenr",
+ "data_set_field": "housenr"
+ },
+ {
+ "transform_field": "stateCode",
+ "data_set_field": "stateCode"
+ },
+ {
+ "transform_field": "state",
+ "data_set_field": "state"
+ }
+ ],
+ "field_order": [
+ "id"
+ ],
+ "transform_name": "Validate",
+ "data_set_name": "0008-target-output"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0008-target-output-transform-validation UNIT",
+ "description": "",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename": "./0008-target-output-transform-validation.hpl",
+ "test_type": "UNIT_TEST"
+}
\ No newline at end of file
diff --git a/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java b/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java
index 9b0dce2..8197c1f 100644
--- a/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java
+++ b/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java
@@ -248,7 +248,7 @@
//
Map<String, PCollection<HopRow>> transformCollectionMap = new HashMap<>();
- // Handle io
+ // Handle input
//
handleBeamInputTransforms(log, transformCollectionMap, pipeline);
@@ -374,14 +374,31 @@
}
TransformMeta previousTransform = previousTransforms.get(0);
- PCollection<HopRow> input = transformCollectionMap.get(previousTransform.getName());
+ // See if this output transform isn't targeted specifically by the previous transform.
+ //
+ // Check in the map to see if previousTransform isn't targeting this one
+ //
+ String targetName =
+ HopBeamUtil.createTargetTupleId(
+ previousTransform.getName(), transformMeta.getName());
+ PCollection<HopRow> input = transformCollectionMap.get(targetName);
if (input == null) {
- throw new HopException(
- "Previous PCollection for transform "
- + previousTransform.getName()
- + " could not be found");
+ input = transformCollectionMap.get(previousTransform.getName());
+ if (input == null) {
+ throw new HopException(
+ "Previous PCollection for transform "
+ + previousTransform.getName()
+ + " could not be found");
+ }
+ } else {
+ log.logBasic(
+ "Transform "
+ + transformMeta.getName()
+ + " reading from previous transform targeting this one using : "
+ + targetName);
}
+
// What fields are we getting from the previous transform(s)?
//
IRowMeta rowMeta = pipelineMeta.getTransformFields(variables, previousTransform);
diff --git a/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/HopGuiFlagPipelineForUnitTestExtensionPoint.java b/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/HopGuiFlagPipelineForUnitTestExtensionPoint.java
index 3b23d05..a217d34 100644
--- a/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/HopGuiFlagPipelineForUnitTestExtensionPoint.java
+++ b/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/HopGuiFlagPipelineForUnitTestExtensionPoint.java
@@ -23,7 +23,9 @@
import org.apache.hop.core.logging.ILogChannel;
import org.apache.hop.core.util.StringUtil;
import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration;
import org.apache.hop.pipeline.engine.IPipelineEngine;
+import org.apache.hop.pipeline.engines.local.LocalPipelineRunConfiguration;
import org.apache.hop.testing.PipelineUnitTest;
import org.apache.hop.testing.gui.TestingGuiPlugin;
import org.apache.hop.testing.util.DataSetConst;
@@ -45,6 +47,13 @@
public void callExtensionPoint(ILogChannel log, IVariables variables, IPipelineEngine pipeline)
throws HopException {
+ IPipelineEngineRunConfiguration runConfig =
+ pipeline.getPipelineRunConfiguration().getEngineRunConfiguration();
+ if (!(runConfig instanceof LocalPipelineRunConfiguration)) {
+ pipeline.setVariable(DataSetConst.VAR_RUN_UNIT_TEST, "N");
+ return;
+ }
+
PipelineUnitTest unitTest = TestingGuiPlugin.getCurrentUnitTest(pipeline.getPipelineMeta());
if (unitTest == null) {
return;
