pinot-tools/pom.xml

<?xml version="1.0"?>
<!--

    Licensed to the Apache Software Foundation (ASF) under one
    or more contributor license agreements.  See the NOTICE file
    distributed with this work for additional information
    regarding copyright ownership.  The ASF licenses this file
    to you under the Apache License, Version 2.0 (the
    "License"); you may not use this file except in compliance
    with the License.  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing,
    software distributed under the License is distributed on an
    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    KIND, either express or implied.  See the License for the
    specific language governing permissions and limitations
    under the License.

-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <parent>
    <artifactId>pinot</artifactId>
    <groupId>org.apache.pinot</groupId>
    <version>0.12.1</version>
    <relativePath>..</relativePath>
  </parent>
  <artifactId>pinot-tools</artifactId>
  <name>Pinot Tools</name>
  <url>https://pinot.apache.org/</url>
  <properties>
    <pinot.root>${basedir}/..</pinot.root>
    <aws.version>2.14.28</aws.version>
    <scala.version>2.12</scala.version>
    <spark.version>3.2.1</spark.version>
  </properties>
  <dependencies>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-core</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-common</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-server</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-controller</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-broker</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-minion</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-batch-ingestion-common</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-avro</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-csv</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-protobuf</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-json</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-orc</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <scope>compile</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-hdfs</artifactId>
      <scope>compile</scope>
      <exclusions>
        <exclusion>
          <groupId>xml-apis</groupId>
          <artifactId>xml-apis</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>xml-apis</groupId>
      <artifactId>xml-apis</artifactId>
      <scope>compile</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-parquet</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-thrift</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-kinesis</artifactId>
      <version>${project.version}</version>
      <scope>runtime</scope>
      <exclusions>
        <exclusion>
          <groupId>software.amazon.awssdk</groupId>
          <artifactId>http-client-spi</artifactId>
        </exclusion>
        <exclusion>
          <groupId>software.amazon.awssdk</groupId>
          <artifactId>protocol-core</artifactId>
        </exclusion>
        <exclusion>
          <groupId>software.amazon.awssdk</groupId>
          <artifactId>annotations</artifactId>
        </exclusion>
        <exclusion>
          <groupId>software.amazon.awssdk</groupId>
          <artifactId>utils</artifactId>
        </exclusion>
        <exclusion>
          <groupId>software.amazon.awssdk</groupId>
          <artifactId>profiles</artifactId>
        </exclusion>
        <exclusion>
          <groupId>software.amazon.awssdk</groupId>
          <artifactId>regions</artifactId>
        </exclusion>
        <exclusion>
          <groupId>software.amazon.awssdk</groupId>
          <artifactId>auth</artifactId>
        </exclusion>
        <exclusion>
          <groupId>software.amazon.awssdk</groupId>
          <artifactId>aws-core</artifactId>
        </exclusion>
        <exclusion>
          <groupId>software.amazon.awssdk</groupId>
          <artifactId>apache-client</artifactId>
        </exclusion>
        <exclusion>
          <groupId>software.amazon.awssdk</groupId>
          <artifactId>sdk-core</artifactId>
        </exclusion>
        <exclusion>
          <groupId>software.amazon.awssdk</groupId>
          <artifactId>netty-nio-client</artifactId>
        </exclusion>
        <exclusion>
          <groupId>io.netty</groupId>
          <artifactId>netty-codec</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>sdk-core</artifactId>
      <version>${aws.version}</version>
    </dependency>
    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>s3</artifactId>
      <version>${aws.version}</version>
    </dependency>
    <dependency>
      <groupId>org.reactivestreams</groupId>
      <artifactId>reactive-streams</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-kafka-${kafka.version}</artifactId>
      <version>${project.version}</version>
      <scope>runtime</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-batch-ingestion-standalone</artifactId>
      <version>${project.version}</version>
      <scope>runtime</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-s3</artifactId>
      <version>${project.version}</version>
      <scope>runtime</scope>
      <exclusions>
        <exclusion>
          <groupId>software.amazon.awssdk</groupId>
          <artifactId>s3</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-minion-builtin-tasks</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-yammer</artifactId>
      <version>${project.version}</version>
      <scope>runtime</scope>
    </dependency>
    <dependency>
      <groupId>commons-cli</groupId>
      <artifactId>commons-cli</artifactId>
    </dependency>
    <dependency>
      <groupId>info.picocli</groupId>
      <artifactId>picocli</artifactId>
    </dependency>
    <dependency>
      <groupId>org.testng</groupId>
      <artifactId>testng</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.glassfish.jersey.containers</groupId>
      <artifactId>jersey-container-grizzly2-http</artifactId>
    </dependency>
    <dependency>
      <groupId>org.glassfish.tyrus.bundles</groupId>
      <artifactId>tyrus-standalone-client</artifactId>
    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-databind</artifactId>
    </dependency>
    <dependency>
      <groupId>org.yaml</groupId>
      <artifactId>snakeyaml</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.logging.log4j</groupId>
      <artifactId>log4j-core</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.commons</groupId>
      <artifactId>commons-math3</artifactId>
    </dependency>
    <dependency>
      <groupId>org.mockito</groupId>
      <artifactId>mockito-core</artifactId>
      <scope>test</scope>
    </dependency>

    <!--
      This dependency is needed for LaunchSparkDataIngestionJobCommand.
      The dependency only contains a few classes and scala library which has been excluded.
      Hence, it will not interfere with spark-core classes present in runtime env
      and will use the env spark version to actually execute the spark job
    -->
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-launcher_${scala.version}</artifactId>
      <version>${spark.version}</version>
      <exclusions>
        <exclusion>
          <groupId>org.scala-lang</groupId>
          <artifactId>scala-library</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
  </dependencies>
  <build>
    <plugins>
      <plugin>
        <groupId>org.codehaus.mojo</groupId>
        <artifactId>appassembler-maven-plugin</artifactId>
        <configuration>
          <programs>
            <program>
              <mainClass>org.apache.pinot.tools.PinotToolLauncher</mainClass>
              <name>pinot-tools</name>
              <jvmSettings>
                <initialMemorySize>1G</initialMemorySize>
                <maxMemorySize>1G</maxMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/pinot-tools-log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.admin.PinotAdministrator</mainClass>
              <name>pinot-admin</name>
              <jvmSettings>
                <initialMemorySize>4G</initialMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/log4j2.xml</extraArgument>
                  <extraArgument>-Dpinot.admin.system.exit=true</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.StarTreeIndexViewer</mainClass>
              <name>star-tree-index-viewer</name>
              <jvmSettings>
                <initialMemorySize>1G</initialMemorySize>
                <maxMemorySize>1G</maxMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/pinot-tools-log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.admin.PinotController</mainClass>
              <name>start-controller</name>
              <jvmSettings>
                <initialMemorySize>1G</initialMemorySize>
                <maxMemorySize>1G</maxMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.admin.PinotBroker</mainClass>
              <name>start-broker</name>
              <jvmSettings>
                <initialMemorySize>1G</initialMemorySize>
                <maxMemorySize>1G</maxMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.admin.PinotServer</mainClass>
              <name>start-server</name>
              <jvmSettings>
                <initialMemorySize>1G</initialMemorySize>
                <maxMemorySize>1G</maxMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.admin.PinotMinion</mainClass>
              <name>start-minion</name>
              <jvmSettings>
                <initialMemorySize>1G</initialMemorySize>
                <maxMemorySize>1G</maxMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.admin.PinotServiceManagerStarter</mainClass>
              <name>start-service-manager</name>
              <jvmSettings>
                <initialMemorySize>1G</initialMemorySize>
                <maxMemorySize>1G</maxMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.Quickstart</mainClass>
              <name>quick-start-batch</name>
              <jvmSettings>
                <initialMemorySize>4G</initialMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/quickstart-log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.HybridQuickstart</mainClass>
              <name>quick-start-hybrid</name>
              <jvmSettings>
                <initialMemorySize>4G</initialMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/quickstart-log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.RealtimeQuickStart</mainClass>
              <name>quick-start-streaming</name>
              <jvmSettings>
                <initialMemorySize>4G</initialMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/quickstart-log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.UpsertQuickStart</mainClass>
              <name>quick-start-upsert-streaming</name>
              <jvmSettings>
                <initialMemorySize>4G</initialMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/quickstart-log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.PartialUpsertQuickStart</mainClass>
              <name>quick-start-partial-upsert-streaming</name>
              <jvmSettings>
                <initialMemorySize>1G</initialMemorySize>
                <maxMemorySize>1G</maxMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/quickstart-log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.JsonIndexQuickStart</mainClass>
              <name>quick-start-json-index-batch</name>
              <jvmSettings>
                <initialMemorySize>4G</initialMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/quickstart-log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.RealtimeJsonIndexQuickStart</mainClass>
              <name>quick-start-json-index-streaming</name>
              <jvmSettings>
                <initialMemorySize>4G</initialMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/quickstart-log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.RealtimeComplexTypeHandlingQuickStart</mainClass>
              <name>quick-start-complex-type-handling-streaming</name>
              <jvmSettings>
                <initialMemorySize>4G</initialMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/quickstart-log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.OfflineComplexTypeHandlingQuickStart</mainClass>
              <name>quick-start-complex-type-handling-offline</name>
              <jvmSettings>
                <initialMemorySize>4G</initialMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/quickstart-log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.AuthQuickstart</mainClass>
              <name>quick-start-auth</name>
              <jvmSettings>
                <initialMemorySize>4G</initialMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/quickstart-log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.AuthZkBasicQuickstart</mainClass>
              <name>quick-start-auth-zk</name>
              <jvmSettings>
                <initialMemorySize>4G</initialMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/quickstart-log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.perf.QueryRunner</mainClass>
              <name>query-runner</name>
              <jvmSettings>
                <initialMemorySize>4G</initialMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/pinot-tools-log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
            <program>
              <mainClass>org.apache.pinot.tools.admin.command.FileSystemCommand</mainClass>
              <name>pinot-fs-util</name>
              <jvmSettings>
                <initialMemorySize>4G</initialMemorySize>
                <extraArguments>
                  <extraArgument>-Dlog4j2.configurationFile=conf/pinot-tools-log4j2.xml</extraArgument>
                </extraArguments>
              </jvmSettings>
            </program>
          </programs>
          <binFileExtensions>
            <unix>.sh</unix>
          </binFileExtensions>

          <!-- Set the target configuration directory to be used in the bin scripts -->
          <configurationDirectory>conf</configurationDirectory>
          <!-- Copy the contents from "/src/main/config" to the target configuration directory in the assembled application -->
          <copyConfigurationDirectory>false</copyConfigurationDirectory>
          <!-- Include the target configuration directory in the beginning of the classpath declaration in the bin scripts -->
          <includeConfigurationDirectoryInClasspath>false</includeConfigurationDirectoryInClasspath>
          <assembleDirectory>${project.build.directory}/${project.artifactId}-pkg</assembleDirectory>
          <unixScriptTemplate>${project.basedir}/src/main/resources/appAssemblerScriptTemplate</unixScriptTemplate>
          <!-- Generate bin scripts for windows and unix pr default -->
          <platforms>
            <platform>unix</platform>
          </platforms>
          <repositoryLayout>flat</repositoryLayout>
          <repositoryName>lib</repositoryName>
        </configuration>
      </plugin>
      <plugin>
        <artifactId>maven-assembly-plugin</artifactId>
        <executions>
          <execution>
            <id>build-pinot-tool-launcher</id>
            <configuration>
              <descriptorRefs>
                <descriptorRef>jar-with-dependencies</descriptorRef>
              </descriptorRefs>
              <archive>
                <manifest>
                  <mainClass>org.apache.pinot.tools.PinotToolLauncher</mainClass>
                </manifest>
              </archive>
              <finalName>pinot-tool-launcher</finalName>
            </configuration>
            <phase>package</phase>
            <goals>
              <goal>single</goal>
            </goals>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-enforcer-plugin</artifactId>
      </plugin>
      <plugin>
        <artifactId>maven-resources-plugin</artifactId>
        <version>2.6</version>
        <executions>
          <execution>
            <id>copy-kafka-resources</id>
            <phase>process-resources</phase>
            <goals>
              <goal>copy-resources</goal>
            </goals>
            <configuration>
              <!-- this is important -->
              <overwrite>true</overwrite>
              <!-- target -->
              <outputDirectory>${basedir}/target/classes/sample_data/</outputDirectory>
              <resources>
                <resource>
                  <!-- source -->
                  <directory>src/main/resources/sample_data/kafka_${kafka.version}</directory>
                </resource>
              </resources>
            </configuration>
          </execution>
          <execution>
            <id>copy-batch-ingestion-resources</id>
            <phase>process-resources</phase>
            <goals>
              <goal>copy-resources</goal>
            </goals>
            <configuration>
              <!-- this is important -->
              <overwrite>true</overwrite>
              <!-- target -->
              <outputDirectory>${basedir}/target/classes/examples/</outputDirectory>
              <resources>
                <resource>
                  <!-- source -->
                  <directory>src/main/resources/examples</directory>
                </resource>
              </resources>
            </configuration>
          </execution>
        </executions>
      </plugin>
    </plugins>
  </build>
</project>