Updated Terraform configuration for recent 4.0.0 changes (#313)
diff --git a/contrib/terraform-testing-infrastructure/README.md b/contrib/terraform-testing-infrastructure/README.md
index 72e1b55..d84989c 100644
--- a/contrib/terraform-testing-infrastructure/README.md
+++ b/contrib/terraform-testing-infrastructure/README.md
@@ -161,7 +161,7 @@
| accumulo\_root\_password | The password for the accumulo root user. A randomly generated password will be used if none is specified here. | `string` | `null` | no |
| accumulo\_testing\_branch\_name | The name of the branch to build and install | `string` | `"main"` | no |
| accumulo\_testing\_repo | URL of the Accumulo Testing git repo | `string` | `"https://github.com/apache/accumulo-testing.git"` | no |
-| accumulo\_version | The branch of Accumulo to download and install | `string` | `"2.1.0-SNAPSHOT"` | no |
+| accumulo\_version | The branch of Accumulo to download and install | `string` | `"4.0.0-SNAPSHOT"` | no |
| ami\_name\_pattern | The pattern of the name of the AMI to use | `any` | n/a | yes |
| ami\_owner | The id of the AMI owner | `any` | n/a | yes |
| authorized\_ssh\_key\_files | List of SSH public key files for the developers that will log into the cluster | `list(string)` | `[]` | no |
@@ -169,11 +169,11 @@
| cloudinit\_merge\_type | Describes the merge behavior for overlapping config blocks in cloud-init. | `string` | `null` | no |
| create\_route53\_records | Indicates whether or not route53 records will be created | `bool` | `false` | no |
| hadoop\_dir | The Hadoop directory on each EC2 node | `string` | `"/data/hadoop"` | no |
-| hadoop\_version | The version of Hadoop to download and install | `string` | `"3.3.4"` | no |
+| hadoop\_version | The version of Hadoop to download and install | `string` | `"3.5.0"` | no |
| instance\_count | The number of EC2 instances to create | `string` | `"2"` | no |
| instance\_type | The type of EC2 instances to create | `string` | `"m5.2xlarge"` | no |
| local\_sources\_dir | Directory on local machine that contains Maven, ZooKeeper or Hadoop binary distributions or Accumulo source tarball | `string` | `""` | no |
-| maven\_version | The version of Maven to download and install | `string` | `"3.8.8"` | no |
+| maven\_version | The version of Maven to download and install | `string` | `"3.9.16"` | no |
| optional\_cloudinit\_config | An optional config block for the cloud-init script. If you set this, you should consider setting cloudinit\_merge\_type to handle merging with the default script as you need. | `string` | `null` | no |
| private\_network | Indicates whether or not the user is on a private network and access to hosts should be through the private IP addresses rather than public ones. | `bool` | `false` | no |
| root\_volume\_gb | The size, in GB, of the EC2 instance root volume | `string` | `"300"` | no |
@@ -183,7 +183,7 @@
| us\_east\_1b\_subnet | The AWS subnet id for the us-east-1b subnet | `any` | n/a | yes |
| us\_east\_1e\_subnet | The AWS subnet id for the us-east-1e subnet | `any` | n/a | yes |
| zookeeper\_dir | The ZooKeeper directory on each EC2 node | `string` | `"/data/zookeeper"` | no |
-| zookeeper\_version | The version of ZooKeeper to download and install | `string` | `"3.8.0"` | no |
+| zookeeper\_version | The version of ZooKeeper to download and install | `string` | `"3.9.5"` | no |
The following outputs are returned by the `aws` Terraform configuration.
@@ -330,11 +330,11 @@
and builds the software using Maven, then untars the binary tarball to
`${software_root}/accumulo/accumulo-${accumulo_version}`
5. Downloads the [OpenTelemetry](https://opentelemetry.io/) Java Agent jar file and copies it to
- `${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.26.1.jar`
+ `${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.31.1.jar`
6. Copies the Accumulo `test` jar to `${software_root}/accumulo/accumulo-${accumulo_version}/lib`
so that `org.apache.accumulo.test.metrics.TestStatsDRegistryFactory` is on the classpath
7. Downloads the [Micrometer](https://micrometer.io/) StatsD Registry jar file and copies it to
- `${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.12.1.jar`
+ `${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.16.7.jar`
8. Clones, if necessary, the Apache Accumulo Testing Git repo from `${accumulo_testing_repo}`
into `${software_root}/sources/accumulo-testing-repo`. It switches to the
`${accumulo_testing_branch_name}` branch and builds the software using Maven.
diff --git a/contrib/terraform-testing-infrastructure/aws/main.tf b/contrib/terraform-testing-infrastructure/aws/main.tf
index 3719b20..8e68c48 100644
--- a/contrib/terraform-testing-infrastructure/aws/main.tf
+++ b/contrib/terraform-testing-infrastructure/aws/main.tf
@@ -69,6 +69,7 @@
key = "accumulo-testing/terraform.tfstate"
region = "us-east-1"
dynamodb_table = "accumulo-testing-tf-locks"
+ use_lockfile = true
encrypt = true
}
}
diff --git a/contrib/terraform-testing-infrastructure/aws/variables.tf b/contrib/terraform-testing-infrastructure/aws/variables.tf
index 3100f12..6e53953 100644
--- a/contrib/terraform-testing-infrastructure/aws/variables.tf
+++ b/contrib/terraform-testing-infrastructure/aws/variables.tf
@@ -131,25 +131,25 @@
}
variable "maven_version" {
- default = "3.8.8"
+ default = "3.9.16"
description = "The version of Maven to download and install"
nullable = false
}
variable "zookeeper_version" {
- default = "3.8.0"
+ default = "3.9.5"
description = "The version of ZooKeeper to download and install"
nullable = false
}
variable "hadoop_version" {
- default = "3.3.4"
+ default = "3.5.0"
description = "The version of Hadoop to download and install"
nullable = false
}
variable "accumulo_version" {
- default = "2.1.0-SNAPSHOT"
+ default = "4.0.0-SNAPSHOT"
description = "The branch of Accumulo to download and install"
nullable = false
}
diff --git a/contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl b/contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl
index 55a9ed4..ee1ff9f 100644
--- a/contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl
+++ b/contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl
@@ -60,6 +60,7 @@
- java-11-openjdk-devel
%{ if centos7 == "false" ~}
- java-17-openjdk-devel
+ - java-21-openjdk-devel
%{ endif ~}
- git
%{ endif ~}
diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/main.tf b/contrib/terraform-testing-infrastructure/modules/config-files/main.tf
index 291a277..71146a9 100644
--- a/contrib/terraform-testing-infrastructure/modules/config-files/main.tf
+++ b/contrib/terraform-testing-infrastructure/modules/config-files/main.tf
@@ -56,6 +56,7 @@
java_11_home = var.os_distro == "ubuntu" ? "/usr/lib/jvm/java-11-openjdk-amd64" : "/usr/lib/jvm/java-11-openjdk"
java_17_home = var.os_distro == "ubuntu" ? "/usr/lib/jvm/java-17-openjdk-amd64" : "/usr/lib/jvm/java-17-openjdk"
+ java_21_home = var.os_distro == "ubuntu" ? "/usr/lib/jvm/java-21-openjdk-amd64" : "/usr/lib/jvm/java-21-openjdk"
accumulo_root_pw = coalesce(var.accumulo_root_password, random_string.accumulo_root_password.result)
template_vars = {
@@ -64,6 +65,7 @@
worker_ips = var.worker_ips
java_11_home = local.java_11_home
java_17_home = local.java_17_home
+ java_21_home = local.java_21_home
accumulo_branch_name = var.accumulo_branch_name
accumulo_dir = var.accumulo_dir
accumulo_repo = var.accumulo_repo
diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/accumulo-properties.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/accumulo-properties.tftpl
index 1eda8f9..8d9c051 100644
--- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/accumulo-properties.tftpl
+++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/accumulo-properties.tftpl
@@ -41,7 +41,7 @@
# Micrometer settings
general.micrometer.enabled=true
general.micrometer.jvm.metrics.enabled=true
-general.micrometer.factory=org.apache.accumulo.test.metrics.TestStatsDRegistryFactory
+general.micrometer.factory=org.apache.accumulo.core.spi.metrics.AccumuloMonitorMeterRegistryFactory,org.apache.accumulo.test.metrics.TestStatsDRegistryFactory
# Per-Table Encryption
instance.crypto.opts.factory=org.apache.accumulo.core.spi.crypto.PerTableCryptoServiceFactory
diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/cluster.yaml.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/cluster.yaml.tftpl
index e347c75..91dc12f 100644
--- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/cluster.yaml.tftpl
+++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/cluster.yaml.tftpl
@@ -18,7 +18,9 @@
#
manager:
- - ${manager_ip}
+ servers_per_host: 1
+ hosts:
+ - ${manager_ip}
monitor:
- ${manager_ip}
diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-datanode.service.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-datanode.service.tftpl
index 5689456..73707f4 100644
--- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-datanode.service.tftpl
+++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-datanode.service.tftpl
@@ -22,7 +22,7 @@
After=remote-fs.target
[Service]
-Environment=JAVA_HOME=${java_11_home}
+Environment=JAVA_HOME=${java_17_home}
Environment=HADOOP_HOME=${software_root}/hadoop/hadoop-${hadoop_version}
Environment=HADOOP_LOG_DIR=${hadoop_dir}/logs
User=hadoop
diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-namenode.service.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-namenode.service.tftpl
index 6d4dafe..bdd441c 100644
--- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-namenode.service.tftpl
+++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-namenode.service.tftpl
@@ -22,7 +22,7 @@
After=remote-fs.target
[Service]
-Environment=JAVA_HOME=${java_11_home}
+Environment=JAVA_HOME=${java_17_home}
Environment=HADOOP_HOME=${software_root}/hadoop/hadoop-${hadoop_version}
Environment=HADOOP_LOG_DIR=${hadoop_dir}/logs
User=hadoop
diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl
index d07b2e6..b03a525 100644
--- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl
+++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl
@@ -10,7 +10,7 @@
export M2_HOME=${software_root}/apache-maven/apache-maven-${maven_version}
export PDSH_RCMD_TYPE=ssh
-export ACCUMULO_JAVA_OPTS="-javaagent:${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.26.1.jar -Dotel.traces.exporter=otlp -Dotel.metrics.exporter=none -Dotel.logs.exporter=none -Dotel.exporter.otlp.endpoint=http://${manager_ip}:4317 -Dotel.exporter.otlp.protocol=grpc -Dtest.meter.registry.host=${manager_ip} -Dtest.meter.registry.port=8125"
+export ACCUMULO_JAVA_OPTS="-javaagent:${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.31.1.jar -Dotel.traces.exporter=otlp -Dotel.metrics.exporter=none -Dotel.logs.exporter=none -Dotel.exporter.otlp.endpoint=http://${manager_ip}:4317 -Dotel.exporter.otlp.protocol=grpc -Dtest.meter.registry.host=${manager_ip} -Dtest.meter.registry.port=8125"
# User specific environment and startup programs
PATH=$PATH:$HOME/.local/bin:$HOME/bin:$ZOOKEEPER_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ACCUMULO_HOME/bin:$M2_HOME/bin
diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/initialize_accumulo.sh.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/initialize_accumulo.sh.tftpl
index 4032048..dd3aed4 100755
--- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/initialize_accumulo.sh.tftpl
+++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/initialize_accumulo.sh.tftpl
@@ -40,8 +40,8 @@
#
# Initialize Accumulo with the supplied instance name and root user password
#
-accumulo init --instance-name "$1" --password "$2"
-accumulo init --add-resource-groups accumulo_meta,user_small,user_large
+accumulo inst init --instance-name "$1" --password "$2"
+accumulo inst init --add-resource-groups accumulo_meta,user_small,user_large
#
# Start jaegertracing container
diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl
index bff423d..9dc697e 100644
--- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl
+++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl
@@ -20,7 +20,8 @@
set -eo pipefail
-export JAVA_HOME=${java_17_home}
+# Use Java 21 for building
+export JAVA_HOME=${java_21_home}
SOURCES_DIR="${software_root}/sources"
@@ -88,28 +89,13 @@
else
#
# Download, build, and install Accumulo
- echo "Cloning Accumulo-Access repo from https://github.com/apache/accumulo-access.git"
- cd $SOURCES_DIR
- rm -rf accumulo-access-repo
- git clone https://github.com/apache/accumulo-access.git accumulo-access-repo
- cd accumulo-access-repo
- git checkout main
- ${software_root}/apache-maven/apache-maven-${maven_version}/bin/mvn -ntp clean install -PskipQA
-
echo "Binary tarball not found, cloning Accumulo repo from ${accumulo_repo}"
cd $SOURCES_DIR
rm -rf accumulo-repo
git clone ${accumulo_repo} accumulo-repo
cd accumulo-repo
git checkout ${accumulo_branch_name}
- # Java 17 required for Accumulo build
- OLDPATH="$PATH"
- export JAVA_HOME=${java_17_home}
- export PATH=$JAVA_HOME/bin:$PATH
${software_root}/apache-maven/apache-maven-${maven_version}/bin/mvn -Dapache.snapshots=true -ntp clean install -PskipQA
- # Reset JAVA HOME to Java 17 for the remainder of the script
- export JAVA_HOME=${java_17_home}
- export PATH=$OLDPATH
mkdir -p ${software_root}/accumulo
tar zxf assemble/target/accumulo-${accumulo_version}-bin.tar.gz -C ${software_root}/accumulo
fi
@@ -124,8 +110,8 @@
#
# OpenTelemetry dependencies
#
-if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.26.1.jar ]; then
- wget https://search.maven.org/remotecontent?filepath=io/opentelemetry/javaagent/opentelemetry-javaagent/2.26.1/opentelemetry-javaagent-2.26.1.jar -O ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.26.1.jar
+if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.31.1.jar ]; then
+ wget https://search.maven.org/remotecontent?filepath=io/opentelemetry/javaagent/opentelemetry-javaagent/2.31.1/opentelemetry-javaagent-2.31.1.jar -O ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.31.1.jar
fi
#
# Micrometer dependencies
@@ -137,8 +123,8 @@
echo "accumulo-test-${accumulo_version}.jar not found, metrics won't work..."
fi
fi
-if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.14.5.jar ]; then
- wget https://search.maven.org/remotecontent?filepath=io/micrometer/micrometer-registry-statsd/1.14.5/micrometer-registry-statsd-1.14.5.jar -O ${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.14.5.jar
+if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.16.7.jar ]; then
+ wget https://search.maven.org/remotecontent?filepath=io/micrometer/micrometer-registry-statsd/1.16.7/micrometer-registry-statsd-1.16.7.jar -O ${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.16.7.jar
fi
#
diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-nodemanager.service.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-nodemanager.service.tftpl
index b931806..52b8de1 100644
--- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-nodemanager.service.tftpl
+++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-nodemanager.service.tftpl
@@ -22,7 +22,7 @@
After=hadoop-datanode.service
[Service]
-Environment=JAVA_HOME=${java_11_home}
+Environment=JAVA_HOME=${java_17_home}
Environment=YARN_HOME=${software_root}/hadoop/hadoop-${hadoop_version}
Environment=HADOOP_LOG_DIR=${hadoop_dir}/logs
User=hadoop
diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-resourcemanager.service.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-resourcemanager.service.tftpl
index fa7647e..0efe7f3 100644
--- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-resourcemanager.service.tftpl
+++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-resourcemanager.service.tftpl
@@ -22,7 +22,7 @@
After=hadoop-namenode.service
[Service]
-Environment=JAVA_HOME=${java_11_home}
+Environment=JAVA_HOME=${java_17_home}
Environment=YARN_HOME=${software_root}/hadoop/hadoop-${hadoop_version}
Environment=HADOOP_LOG_DIR=${hadoop_dir}/logs
User=hadoop
diff --git a/contrib/terraform-testing-infrastructure/timely-grafana/Dockerfile b/contrib/terraform-testing-infrastructure/timely-grafana/Dockerfile
index b5386ac..a6b9b68 100644
--- a/contrib/terraform-testing-infrastructure/timely-grafana/Dockerfile
+++ b/contrib/terraform-testing-infrastructure/timely-grafana/Dockerfile
@@ -21,12 +21,12 @@
COPY build_output/maven /opt/apache-maven
COPY build_output/repo /opt/apache-maven
COPY settings.xml /tmp/settings.xml
-ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk
ENV TIMELY_VERSION="3.0.0-SNAPSHOT"
-ARG HADOOP_VERSION="3.3.6"
+ARG HADOOP_VERSION="3.5.0"
RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
- dnf install -y java-17-openjdk-devel git wget collectd collectd-java && \
+ dnf install -y java-21-openjdk-devel git wget collectd collectd-java && \
dnf clean all && \
rm -rf /var/cache/dnf
@@ -38,16 +38,25 @@
/opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml clean install
RUN cd /opt && \
- git clone https://github.com/NationalSecurityAgency/datawave-in-memory-accumulo.git datawave-in-memory-accumulo && \
- cd datawave-in-memory-accumulo && \
- git checkout accumulo4 && \
+ git clone https://github.com/NationalSecurityAgency/datawave.git datawave && \
+ cd datawave && \
+ git checkout c1fd5d0a787bfa73301a586d0e68598426a32ed7 && \
+ cd contrib/datawave-utils/read-properties && \
+ /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean install && \
+ git checkout 7fa6fc2261cc79bce9b468d6ae8b248e44f8ff4f && \
+ cd ../code-style && \
+ /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean install && \
+ git checkout accumulo4-2026 && \
+ cd ../../.. && \
+ /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean install -N && \
+ cd core/in-memory-accumulo && \
/opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean install
RUN cd /opt && \
git clone https://github.com/NationalSecurityAgency/timely.git timely && \
cd timely && \
git checkout accumulo4 && \
- /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean package -Pcollectd -DskipTests
+ /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean package -Pcollectd -DskipTests -Denforcer.skip=true -Dspotbugs.skip=true
RUN mkdir /tmp/timely-server && \
tar zxf /opt/timely/server/target/timely-server-${TIMELY_VERSION}-dist.tar.gz --strip-components=1 -C /tmp/timely-server && \
diff --git a/contrib/terraform-testing-infrastructure/timely-grafana/build-image.sh b/contrib/terraform-testing-infrastructure/timely-grafana/build-image.sh
index 6291972..1fd6f2c 100755
--- a/contrib/terraform-testing-infrastructure/timely-grafana/build-image.sh
+++ b/contrib/terraform-testing-infrastructure/timely-grafana/build-image.sh
@@ -18,7 +18,7 @@
# under the License.
#
-HADOOP_VERSION=${1:-"3.3.6"}
+HADOOP_VERSION=${1:-"3.5.0"}
MVN_REPO=$2
MVN_DIR=$3