integration-tests/transforms/0029-parquet-output.hpl

<?xml version="1.0" encoding="UTF-8"?>
<!--

Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

-->
<pipeline>
  <info>
    <name>0029-parquet-output</name>
    <name_sync_with_filename>Y</name_sync_with_filename>
    <description/>
    <extended_description/>
    <pipeline_version/>
    <pipeline_type>Normal</pipeline_type>
    <parameters>
    </parameters>
    <capture_transform_performance>N</capture_transform_performance>
    <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
    <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
    <created_user>-</created_user>
    <created_date>2021/06/10 15:12:24.183</created_date>
    <modified_user>-</modified_user>
    <modified_date>2021/06/10 15:12:24.183</modified_date>
    <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
    <is_key_private>N</is_key_private>
  </info>
  <notepads>
  </notepads>
  <order>
    <hop>
      <from>customers-100.txt</from>
      <to>${java.io.tmpdir}/customers.parquet.snappy</to>
      <enabled>Y</enabled>
    </hop>
  </order>
  <transform>
    <name>${java.io.tmpdir}/customers.parquet.snappy</name>
    <type>ParquetFileOutput</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <compression_codec>SNAPPY</compression_codec>
    <data_page_size>1048576</data_page_size>
    <dictionary_page_size>1048576</dictionary_page_size>
    <fields>
      <field>
        <source_field>id</source_field>
        <target_field>id</target_field>
      </field>
      <field>
        <source_field>name</source_field>
        <target_field>name</target_field>
      </field>
      <field>
        <source_field>firstname</source_field>
        <target_field>firstname</target_field>
      </field>
      <field>
        <source_field>zip</source_field>
        <target_field>zip</target_field>
      </field>
      <field>
        <source_field>city</source_field>
        <target_field>city</target_field>
      </field>
      <field>
        <source_field>birthdate</source_field>
        <target_field>birthdate</target_field>
      </field>
      <field>
        <source_field>street</source_field>
        <target_field>street</target_field>
      </field>
      <field>
        <source_field>housenr</source_field>
        <target_field>housenr</target_field>
      </field>
      <field>
        <source_field>stateCode</source_field>
        <target_field>stateCode</target_field>
      </field>
      <field>
        <source_field>state</source_field>
        <target_field>state</target_field>
      </field>
    </fields>
    <filename_split_size>1000000</filename_split_size>
    <filename_base>${java.io.tmpdir}/it/parquet/customers</filename_base>
    <filename_create_parent_folders>Y</filename_create_parent_folders>
    <filename_datetime_format>yyyyMMdd-HHmmss</filename_datetime_format>
    <filename_ext>parquet</filename_ext>
    <filename_include_copy>N</filename_include_copy>
    <filename_include_date>N</filename_include_date>
    <filename_include_datetime>N</filename_include_datetime>
    <filename_include_split>N</filename_include_split>
    <filename_include_time>N</filename_include_time>
    <row_group_size>20000</row_group_size>
    <version>1.0</version>
    <attributes/>
    <GUI>
      <xloc>384</xloc>
      <yloc>144</yloc>
    </GUI>
  </transform>
  <transform>
    <name>customers-100.txt</name>
    <type>CSVInput</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <filename>${PROJECT_HOME}/files/customers-100.txt</filename>
    <filename_field/>
    <rownum_field/>
    <include_filename>N</include_filename>
    <separator>;</separator>
    <enclosure>"</enclosure>
    <header>Y</header>
    <buffer_size>50000</buffer_size>
    <lazy_conversion>Y</lazy_conversion>
    <add_filename_result>N</add_filename_result>
    <parallel>N</parallel>
    <newline_possible>N</newline_possible>
    <encoding/>
    <fields>
      <field>
        <name>id</name>
        <type>Integer</type>
        <format> #</format>
        <currency>$</currency>
        <decimal>.</decimal>
        <group>,</group>
        <length>15</length>
        <precision>0</precision>
        <trim_type>none</trim_type>
      </field>
      <field>
        <name>name</name>
        <type>String</type>
        <format/>
        <currency>$</currency>
        <decimal>.</decimal>
        <group>,</group>
        <length>10</length>
        <precision>-1</precision>
        <trim_type>none</trim_type>
      </field>
      <field>
        <name>firstname</name>
        <type>String</type>
        <format/>
        <currency>$</currency>
        <decimal>.</decimal>
        <group>,</group>
        <length>13</length>
        <precision>-1</precision>
        <trim_type>none</trim_type>
      </field>
      <field>
        <name>zip</name>
        <type>Integer</type>
        <format> #</format>
        <currency>$</currency>
        <decimal>.</decimal>
        <group>,</group>
        <length>15</length>
        <precision>0</precision>
        <trim_type>none</trim_type>
      </field>
      <field>
        <name>city</name>
        <type>String</type>
        <format/>
        <currency>$</currency>
        <decimal>.</decimal>
        <group>,</group>
        <length>8</length>
        <precision>-1</precision>
        <trim_type>none</trim_type>
      </field>
      <field>
        <name>birthdate</name>
        <type>Date</type>
        <format>yyyy/MM/dd</format>
        <currency>$</currency>
        <decimal>.</decimal>
        <group>,</group>
        <length>-1</length>
        <precision>-1</precision>
        <trim_type>none</trim_type>
      </field>
      <field>
        <name>street</name>
        <type>String</type>
        <format/>
        <currency>$</currency>
        <decimal>.</decimal>
        <group>,</group>
        <length>11</length>
        <precision>-1</precision>
        <trim_type>none</trim_type>
      </field>
      <field>
        <name>housenr</name>
        <type>Integer</type>
        <format> #</format>
        <currency>$</currency>
        <decimal>.</decimal>
        <group>,</group>
        <length>15</length>
        <precision>0</precision>
        <trim_type>none</trim_type>
      </field>
      <field>
        <name>stateCode</name>
        <type>String</type>
        <format/>
        <currency>$</currency>
        <decimal>.</decimal>
        <group>,</group>
        <length>9</length>
        <precision>-1</precision>
        <trim_type>none</trim_type>
      </field>
      <field>
        <name>state</name>
        <type>String</type>
        <format/>
        <currency>$</currency>
        <decimal>.</decimal>
        <group>,</group>
        <length>30</length>
        <precision>-1</precision>
        <trim_type>none</trim_type>
      </field>
    </fields>
    <attributes/>
    <GUI>
      <xloc>160</xloc>
      <yloc>144</yloc>
    </GUI>
  </transform>
  <transform_error_handling>
  </transform_error_handling>
  <attributes/>
</pipeline>