contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl

#! /usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

set -eo pipefail

export JAVA_HOME=${java_11_home}

SOURCES_DIR="${software_root}/sources"

cd ${software_root}
mkdir -p $SOURCES_DIR

#
# Download and configure Maven
#
MVN_URL="https://dlcdn.apache.org/maven/maven-3/${maven_version}/binaries/apache-maven-${maven_version}-bin.tar.gz"
MVN_SRC="$${SOURCES_DIR}/apache-maven-${maven_version}-bin.tar.gz"

if [ ! -f $MVN_SRC ]; then
  wget $MVN_URL -O $MVN_SRC
fi
if [ ! -d ${software_root}/apache-maven/apache-maven-${maven_version} ]; then
  mkdir -p ${software_root}/apache-maven
  tar zxf $MVN_SRC -C ${software_root}/apache-maven
  [ -d ~/.m2 ] || mkdir ~/.m2
  cat << 'END' >> ~/.m2/settings.xml
  <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd">
      <localRepository>${software_root}/apache-maven/repository</localRepository>
  </settings>
END
fi

#
# Download and Install ZooKeeper
#
ZK_URL="https://dlcdn.apache.org/zookeeper/zookeeper-${zookeeper_version}/apache-zookeeper-${zookeeper_version}-bin.tar.gz"
ZK_SRC="$${SOURCES_DIR}/apache-zookeeper-${zookeeper_version}-bin.tar.gz"

if [ ! -f $ZK_SRC ]; then
  wget $ZK_URL -O $ZK_SRC
fi
if [ ! -d ${software_root}/zookeeper/apache-zookeeper-${zookeeper_version}-bin ]; then
  mkdir -p ${software_root}/zookeeper
  tar zxf $ZK_SRC -C ${software_root}/zookeeper
fi

#
# Download and Install Hadoop
#
HADOOP_URL="https://downloads.apache.org/hadoop/common/hadoop-${hadoop_version}/hadoop-${hadoop_version}.tar.gz"
HADOOP_SRC="$${SOURCES_DIR}/hadoop-${hadoop_version}.tar.gz"

if [ ! -f $HADOOP_SRC ]; then
  wget $HADOOP_URL -O $HADOOP_SRC 
fi
if [ ! -d ${software_root}/hadoop/hadoop-${hadoop_version} ]; then
  mkdir -p ${software_root}/hadoop
  tar zxf $HADOOP_SRC -C ${software_root}/hadoop
fi

#
# If Accumulo binary tarball provided, then untar it and use it
#
ACCUMULO_SRC="$${SOURCES_DIR}/accumulo-${accumulo_version}-bin.tar.gz"

if [ -f $ACCUMULO_SRC ]; then
  echo "Binary tarball found, untarring it..."
  mkdir -p ${software_root}/accumulo
  tar zxf $ACCUMULO_SRC -C ${software_root}/accumulo
else
#
# Download, build, and install Accumulo
  echo "Binary tarball not found, cloning Accumulo repo from ${accumulo_repo}"
  rm -rf $SOURCES_DIR/accumulo-repo
  cd $SOURCES_DIR
  git clone ${accumulo_repo} accumulo-repo
  cd accumulo-repo
  git checkout ${accumulo_branch_name}
  # Java 17 required for Accumulo build
  OLDPATH="$PATH"
  export JAVA_HOME=${java_17_home}
  export PATH=$JAVA_HOME/bin:$PATH
  ${software_root}/apache-maven/apache-maven-${maven_version}/bin/mvn -ntp clean install -PskipQA
  # Reset JAVA HOME to Java 11 for the remainder of the script
  export JAVA_HOME=${java_11_home}
  export PATH=$OLDPATH
  mkdir -p ${software_root}/accumulo
  tar zxf assemble/target/accumulo-${accumulo_version}-bin.tar.gz -C ${software_root}/accumulo
fi
${software_root}/accumulo/accumulo-${accumulo_version}/bin/accumulo-util build-native

#
# Generate a 32-byte encryption key and put into ${software_root}/accumulo/accumulo-${accumulo_version}/conf.
# This will get distributed to all nodes and the properties that use the file are in accumulo.properties
#
openssl rand -out ${software_root}/accumulo/accumulo-${accumulo_version}/conf/per-table-enc-key 32

#
# OpenTelemetry dependencies
#
if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-1.32.0.jar ]; then
  wget https://search.maven.org/remotecontent?filepath=io/opentelemetry/javaagent/opentelemetry-javaagent/1.32.0/opentelemetry-javaagent-1.32.0.jar -O ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-1.32.0.jar
fi
#
# Micrometer dependencies
#
if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/accumulo-test-${accumulo_version}.jar ]; then
  if [ -f $SOURCES_DIR/accumulo-repo/test/target/accumulo-test-${accumulo_version}.jar ]; then
    cp $SOURCES_DIR/accumulo-repo/test/target/accumulo-test-${accumulo_version}.jar ${software_root}/accumulo/accumulo-${accumulo_version}/lib/.
  else
    echo "accumulo-test-${accumulo_version}.jar not found, metrics won't work..."
  fi
fi
if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.12.1.jar ]; then
  wget https://search.maven.org/remotecontent?filepath=io/micrometer/micrometer-registry-statsd/1.12.1/micrometer-registry-statsd-1.12.1.jar -O ${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.12.1.jar
fi

#
# Download and build Accumulo-Testing
#
TESTING_SRC="$${SOURCES_DIR}/accumulo-testing.zip"

cd ${software_root}
if [ -f $TESTING_SRC ]; then
  echo "Accumulo Testing tarball found, untarring it..."
  mkdir -p $SOURCES_DIR/accumulo-testing-repo
  tar zxf $TESTING_SRC -C $SOURCES_DIR/accumulo-testing-repo
else
  # Download, build, and install Accumulo Testing
  rm -rf $SOURCES_DIR/accumulo-testing-repo
  cd $SOURCES_DIR
  git clone ${accumulo_testing_repo} accumulo-testing-repo
  cd accumulo-testing-repo
  git checkout ${accumulo_testing_branch_name}
  ${software_root}/apache-maven/apache-maven-${maven_version}/bin/mvn -ntp clean package -DskipTests -DskipITs
fi

#
# Copy the configuration files to the correct places
#
cp ${software_root}/conf/zoo.cfg ${software_root}/zookeeper/apache-zookeeper-${zookeeper_version}-bin/conf/zoo.cfg
cp ${software_root}/conf/hdfs-site.xml ${software_root}/hadoop/hadoop-${hadoop_version}/etc/hadoop/hdfs-site.xml
cp ${software_root}/conf/core-site.xml ${software_root}/hadoop/hadoop-${hadoop_version}/etc/hadoop/core-site.xml
cp ${software_root}/conf/yarn-site.xml ${software_root}/hadoop/hadoop-${hadoop_version}/etc/hadoop/yarn-site.xml
cp ${software_root}/conf/cluster.yaml ${software_root}/accumulo/accumulo-${accumulo_version}/conf/cluster.yaml
cp ${software_root}/conf/accumulo.properties ${software_root}/accumulo/accumulo-${accumulo_version}/conf/accumulo.properties
cp ${software_root}/conf/accumulo-client.properties ${software_root}/accumulo/accumulo-${accumulo_version}/conf/accumulo-client.properties
mkdir -p ${software_root}/telegraf/conf
cp ${software_root}/conf/telegraf.conf ${software_root}/telegraf/conf/.

# Update configuration properties for accumulo-testing
defaultFS=$(hdfs getconf -confKey fs.defaultFS)
sed -ri "s|^test.common.hdfs.root=.*$|test.common.hdfs.root=$${defaultFS}|" $SOURCES_DIR/accumulo-testing-repo/conf/accumulo-testing.properties
sed -ri "s|^test.common.yarn.resource.manager=.*$|test.common.yarn.resource.manager=${manager_ip}|" $SOURCES_DIR/accumulo-testing-repo/conf/accumulo-testing.properties

#
# Make directories that will be needed for metrics collection
#
sudo mkdir -p /data/metrics/influxdb
sudo mkdir -p /data/metrics/grafana
sudo chown -R hadoop.hadoop /data/metrics
sudo chmod 777 /data/metrics/influxdb

# Sync the software root dir to all of the worker nodes, but exclude accumulo source, maven, and conf dirs.
echo "Sycing ${software_root} to worker nodes..."
tar cf /tmp/accumulo-testing.tar -C ${software_root} \
  --exclude=./conf \
  --exclude=./apache-maven \
  --exclude=./accumulo_repo \
  --exclude=./sources .
pdcp -g worker /tmp/accumulo-testing.tar /tmp/.
pdsh -S -g worker tar xf /tmp/accumulo-testing.tar -C ${software_root}
rm -f /tmp/accumulo-testing.tar
pdsh -S -g worker rm -f /tmp/accumulo-testing.tar