| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| import CommonJobProperties as common |
| import InfluxDBCredentialsHelper |
| |
| def jobs = [ |
| [ |
| name : 'beam_PerformanceTests_TextIOIT', |
| description : 'Runs performance tests for TextIOIT', |
| test : 'org.apache.beam.sdk.io.text.TextIOIT', |
| githubTitle : 'Java TextIO Performance Test', |
| githubTriggerPhrase: 'Run Java TextIO Performance Test', |
| pipelineOptions : [ |
| bigQueryDataset : 'beam_performance', |
| bigQueryTable : 'textioit_results', |
| influxMeasurement : 'textioit_results', |
| numberOfRecords : '25000000', |
| expectedHash : 'f8453256ccf861e8a312c125dfe0e436', |
| datasetSize : '1062290000', |
| numWorkers : '5', |
| autoscalingAlgorithm: 'NONE' |
| ] |
| ], |
| [ |
| name : 'beam_PerformanceTests_Compressed_TextIOIT', |
| description : 'Runs performance tests for TextIOIT with GZIP compression', |
| test : 'org.apache.beam.sdk.io.text.TextIOIT', |
| githubTitle : 'Java CompressedTextIO Performance Test', |
| githubTriggerPhrase: 'Run Java CompressedTextIO Performance Test', |
| pipelineOptions : [ |
| bigQueryDataset : 'beam_performance', |
| bigQueryTable : 'compressed_textioit_results', |
| influxMeasurement : 'compressed_textioit_results', |
| numberOfRecords : '450000000', |
| expectedHash : '8a3de973354abc6fba621c6797cc0f06', |
| datasetSize : '1097840000', |
| compressionType : 'GZIP', |
| numWorkers : '5', |
| autoscalingAlgorithm: 'NONE' |
| ] |
| ], |
| [ |
| name : 'beam_PerformanceTests_ManyFiles_TextIOIT', |
| description : 'Runs performance tests for TextIOIT with many output files', |
| test : 'org.apache.beam.sdk.io.text.TextIOIT', |
| githubTitle : 'Java ManyFilesTextIO Performance Test', |
| githubTriggerPhrase: 'Run Java ManyFilesTextIO Performance Test', |
| pipelineOptions : [ |
| bigQueryDataset : 'beam_performance', |
| bigQueryTable : 'many_files_textioit_results', |
| influxMeasurement : 'many_files_textioit_results', |
| reportGcsPerformanceMetrics: 'true', |
| gcsPerformanceMetrics : 'true', |
| numberOfRecords : '25000000', |
| expectedHash : 'f8453256ccf861e8a312c125dfe0e436', |
| datasetSize : '1062290000', |
| numberOfShards : '1000', |
| numWorkers : '5', |
| autoscalingAlgorithm : 'NONE' |
| ] |
| |
| ], |
| [ |
| name : 'beam_PerformanceTests_AvroIOIT', |
| description : 'Runs performance tests for AvroIOIT', |
| test : 'org.apache.beam.sdk.io.avro.AvroIOIT', |
| githubTitle : 'Java AvroIO Performance Test', |
| githubTriggerPhrase: 'Run Java AvroIO Performance Test', |
| pipelineOptions : [ |
| numberOfRecords : '225000000', |
| expectedHash : '2f9f5ca33ea464b25109c0297eb6aecb', |
| datasetSize : '1089730000', |
| bigQueryDataset : 'beam_performance', |
| bigQueryTable : 'avroioit_results', |
| influxMeasurement : 'avroioit_results', |
| numWorkers : '5', |
| autoscalingAlgorithm: 'NONE' |
| ] |
| ], |
| [ |
| name : 'beam_PerformanceTests_TFRecordIOIT', |
| description : 'Runs performance tests for beam_PerformanceTests_TFRecordIOIT', |
| test : 'org.apache.beam.sdk.io.tfrecord.TFRecordIOIT', |
| githubTitle : 'Java TFRecordIO Performance Test', |
| githubTriggerPhrase: 'Run Java TFRecordIO Performance Test', |
| pipelineOptions : [ |
| bigQueryDataset : 'beam_performance', |
| bigQueryTable : 'tfrecordioit_results', |
| influxMeasurement : 'tfrecordioit_results', |
| numberOfRecords : '18000000', |
| expectedHash : '543104423f8b6eb097acb9f111c19fe4', |
| datasetSize : '1019380000', |
| numWorkers : '5', |
| autoscalingAlgorithm: 'NONE' |
| ] |
| ], |
| [ |
| name : 'beam_PerformanceTests_XmlIOIT', |
| description : 'Runs performance tests for beam_PerformanceTests_XmlIOIT', |
| test : 'org.apache.beam.sdk.io.xml.XmlIOIT', |
| githubTitle : 'Java XmlIOPerformance Test', |
| githubTriggerPhrase: 'Run Java XmlIO Performance Test', |
| pipelineOptions : [ |
| bigQueryDataset : 'beam_performance', |
| bigQueryTable : 'xmlioit_results', |
| influxMeasurement : 'xmlioit_results', |
| numberOfRecords : '12000000', |
| expectedHash : 'b3b717e7df8f4878301b20f314512fb3', |
| datasetSize : '1076590000', |
| charset : 'UTF-8', |
| numWorkers : '5', |
| autoscalingAlgorithm: 'NONE' |
| ] |
| ], |
| [ |
| name : 'beam_PerformanceTests_ParquetIOIT', |
| description : 'Runs performance tests for beam_PerformanceTests_ParquetIOIT', |
| test : 'org.apache.beam.sdk.io.parquet.ParquetIOIT', |
| githubTitle : 'Java ParquetIOPerformance Test', |
| githubTriggerPhrase: 'Run Java ParquetIO Performance Test', |
| pipelineOptions : [ |
| bigQueryDataset : 'beam_performance', |
| bigQueryTable : 'parquetioit_results', |
| influxMeasurement : 'parquetioit_results', |
| numberOfRecords : '225000000', |
| expectedHash : '2f9f5ca33ea464b25109c0297eb6aecb', |
| datasetSize : '1087370000', |
| numWorkers : '5', |
| autoscalingAlgorithm: 'NONE' |
| ] |
| ], |
| [ |
| name : 'beam_PerformanceTests_TextIOIT_HDFS', |
| description : 'Runs performance tests for TextIOIT on HDFS', |
| test : 'org.apache.beam.sdk.io.text.TextIOIT', |
| githubTitle : 'Java TextIO Performance Test on HDFS', |
| githubTriggerPhrase: 'Run Java TextIO Performance Test HDFS', |
| pipelineOptions : [ |
| bigQueryDataset : 'beam_performance', |
| bigQueryTable : 'textioit_hdfs_results', |
| influxMeasurement : 'textioit_hdfs_results', |
| numberOfRecords : '25000000', |
| expectedHash : 'f8453256ccf861e8a312c125dfe0e436', |
| datasetSize : '1062290000', |
| numWorkers : '5', |
| autoscalingAlgorithm: 'NONE' |
| ] |
| |
| ], |
| [ |
| name : 'beam_PerformanceTests_Compressed_TextIOIT_HDFS', |
| description : 'Runs performance tests for TextIOIT with GZIP compression on HDFS', |
| test : 'org.apache.beam.sdk.io.text.TextIOIT', |
| githubTitle : 'Java CompressedTextIO Performance Test on HDFS', |
| githubTriggerPhrase: 'Run Java CompressedTextIO Performance Test HDFS', |
| pipelineOptions : [ |
| bigQueryDataset : 'beam_performance', |
| bigQueryTable : 'compressed_textioit_hdfs_results', |
| influxMeasurement : 'compressed_textioit_hdfs_results', |
| numberOfRecords : '450000000', |
| expectedHash : '8a3de973354abc6fba621c6797cc0f06', |
| datasetSize : '1097840000', |
| compressionType : 'GZIP', |
| numWorkers : '5', |
| autoscalingAlgorithm: 'NONE' |
| ] |
| ], |
| [ |
| name : 'beam_PerformanceTests_ManyFiles_TextIOIT_HDFS', |
| description : 'Runs performance tests for TextIOIT with many output files on HDFS', |
| test : 'org.apache.beam.sdk.io.text.TextIOIT', |
| githubTitle : 'Java ManyFilesTextIO Performance Test on HDFS', |
| githubTriggerPhrase: 'Run Java ManyFilesTextIO Performance Test HDFS', |
| pipelineOptions : [ |
| bigQueryDataset : 'beam_performance', |
| bigQueryTable : 'many_files_textioit_hdfs_results', |
| influxMeasurement : 'many_files_textioit_hdfs_results', |
| reportGcsPerformanceMetrics: 'true', |
| gcsPerformanceMetrics : 'true', |
| numberOfRecords : '25000000', |
| expectedHash : 'f8453256ccf861e8a312c125dfe0e436', |
| datasetSize : '1062290000', |
| numberOfShards : '1000', |
| numWorkers : '5', |
| autoscalingAlgorithm : 'NONE' |
| ] |
| |
| ], |
| [ |
| name : 'beam_PerformanceTests_AvroIOIT_HDFS', |
| description : 'Runs performance tests for AvroIOIT on HDFS', |
| test : 'org.apache.beam.sdk.io.avro.AvroIOIT', |
| githubTitle : 'Java AvroIO Performance Test on HDFS', |
| githubTriggerPhrase: 'Run Java AvroIO Performance Test HDFS', |
| pipelineOptions : [ |
| bigQueryDataset : 'beam_performance', |
| bigQueryTable : 'avroioit_hdfs_results', |
| influxMeasurement : 'avroioit_hdfs_results', |
| numberOfRecords : '225000000', |
| expectedHash : '2f9f5ca33ea464b25109c0297eb6aecb', |
| datasetSize : '1089730000', |
| numWorkers : '5', |
| autoscalingAlgorithm: 'NONE' |
| ] |
| ], |
| [ |
| name : 'beam_PerformanceTests_TFRecordIOIT_HDFS', |
| description : 'Runs performance tests for beam_PerformanceTests_TFRecordIOIT on HDFS', |
| test : 'org.apache.beam.sdk.io.tfrecord.TFRecordIOIT', |
| githubTitle : 'Java TFRecordIO Performance Test on HDFS', |
| githubTriggerPhrase: 'Run Java TFRecordIO Performance Test HDFS', |
| pipelineOptions : [ |
| numberOfRecords : '18000000', |
| expectedHash : '543104423f8b6eb097acb9f111c19fe4', |
| datasetSize : '1019380000', |
| numWorkers : '5', |
| autoscalingAlgorithm: 'NONE' |
| ] |
| ], |
| [ |
| name : 'beam_PerformanceTests_XmlIOIT_HDFS', |
| description : 'Runs performance tests for beam_PerformanceTests_XmlIOIT on HDFS', |
| test : 'org.apache.beam.sdk.io.xml.XmlIOIT', |
| githubTitle : 'Java XmlIOPerformance Test on HDFS', |
| githubTriggerPhrase: 'Run Java XmlIO Performance Test HDFS', |
| pipelineOptions : [ |
| bigQueryDataset : 'beam_performance', |
| bigQueryTable : 'xmlioit_hdfs_results', |
| influxMeasurement : 'xmlioit_hdfs_results', |
| numberOfRecords : '12000000', |
| expectedHash : 'b3b717e7df8f4878301b20f314512fb3', |
| datasetSize : '1076590000', |
| charset : 'UTF-8', |
| numWorkers : '5', |
| autoscalingAlgorithm: 'NONE' |
| ] |
| ], |
| [ |
| name : 'beam_PerformanceTests_ParquetIOIT_HDFS', |
| description : 'Runs performance tests for beam_PerformanceTests_ParquetIOIT on HDFS', |
| test : 'org.apache.beam.sdk.io.parquet.ParquetIOIT', |
| githubTitle : 'Java ParquetIOPerformance Test on HDFS', |
| githubTriggerPhrase: 'Run Java ParquetIO Performance Test HDFS', |
| pipelineOptions : [ |
| bigQueryDataset : 'beam_performance', |
| bigQueryTable : 'parquetioit_hdfs_results', |
| influxMeasurement : 'parquetioit_hdfs_results', |
| numberOfRecords : '225000000', |
| expectedHash : '2f9f5ca33ea464b25109c0297eb6aecb', |
| datasetSize : '1087370000', |
| numWorkers : '5', |
| autoscalingAlgorithm: 'NONE' |
| ] |
| ] |
| ] |
| |
| jobs.findAll { |
| it.name in [ |
| 'beam_PerformanceTests_TextIOIT', |
| 'beam_PerformanceTests_Compressed_TextIOIT', |
| 'beam_PerformanceTests_ManyFiles_TextIOIT', |
| 'beam_PerformanceTests_AvroIOIT', |
| 'beam_PerformanceTests_TFRecordIOIT', |
| 'beam_PerformanceTests_XmlIOIT', |
| 'beam_PerformanceTests_ParquetIOIT' |
| ] |
| }.forEach { testJob -> createGCSFileBasedIOITTestJob(testJob) } |
| |
| private void createGCSFileBasedIOITTestJob(testJob) { |
| job(testJob.name) { |
| description(testJob.description) |
| common.setTopLevelMainJobProperties(delegate) |
| common.enablePhraseTriggeringFromPullRequest(delegate, testJob.githubTitle, testJob.githubTriggerPhrase) |
| common.setAutoJob(delegate, 'H */6 * * *') |
| InfluxDBCredentialsHelper.useCredentials(delegate) |
| additionalPipelineArgs = [ |
| influxDatabase: InfluxDBCredentialsHelper.InfluxDBDatabaseName, |
| influxHost: InfluxDBCredentialsHelper.InfluxDBHostUrl, |
| ] |
| testJob.pipelineOptions.putAll(additionalPipelineArgs) |
| |
| def dataflowSpecificOptions = [ |
| runner : 'DataflowRunner', |
| project : 'apache-beam-testing', |
| tempRoot : 'gs://temp-storage-for-perf-tests', |
| filenamePrefix: "gs://temp-storage-for-perf-tests/${testJob.name}/\${BUILD_ID}/", |
| ] |
| |
| Map allPipelineOptions = dataflowSpecificOptions << testJob.pipelineOptions |
| String runner = "dataflow" |
| String filesystem = "gcs" |
| String testTask = ":sdks:java:io:file-based-io-tests:integrationTest" |
| |
| steps { |
| gradle { |
| rootBuildScriptDir(common.checkoutDir) |
| common.setGradleSwitches(delegate) |
| switches("--info") |
| switches("-DintegrationTestPipelineOptions=\'${common.joinPipelineOptions(allPipelineOptions)}\'") |
| switches("-Dfilesystem=\'${filesystem}\'") |
| switches("-DintegrationTestRunner=\'${runner}\'") |
| tasks("${testTask} --tests ${testJob.test}") |
| } |
| } |
| } |
| } |
| |
| jobs.findAll { |
| it.name in [ |
| 'beam_PerformanceTests_TextIOIT_HDFS', |
| 'beam_PerformanceTests_Compressed_TextIOIT_HDFS', |
| 'beam_PerformanceTests_ManyFiles_TextIOIT_HDFS', |
| // TODO(BEAM-3945) TFRecord performance test is failing only when running on hdfs. |
| // We need to fix this before enabling this job on jenkins. |
| //'beam_PerformanceTests_TFRecordIOIT_HDFS', |
| 'beam_PerformanceTests_AvroIOIT_HDFS', |
| 'beam_PerformanceTests_XmlIOIT_HDFS', |
| 'beam_PerformanceTests_ParquetIOIT_HDFS' |
| ] |
| }.forEach { testJob -> createHDFSFileBasedIOITTestJob(testJob) } |
| |
| private void createHDFSFileBasedIOITTestJob(testJob) { |
| job(testJob.name) { |
| description(testJob.description) |
| common.setTopLevelMainJobProperties(delegate) |
| common.enablePhraseTriggeringFromPullRequest(delegate, testJob.githubTitle, testJob.githubTriggerPhrase) |
| common.setAutoJob(delegate, 'H */6 * * *') |
| InfluxDBCredentialsHelper.useCredentials(delegate) |
| additionalPipelineArgs = [ |
| influxDatabase: InfluxDBCredentialsHelper.InfluxDBDatabaseName, |
| influxHost: InfluxDBCredentialsHelper.InfluxDBHostUrl, |
| ] |
| testJob.pipelineOptions.putAll(additionalPipelineArgs) |
| |
| String namespace = common.getKubernetesNamespace(testJob.name) |
| String kubeconfig = common.getKubeconfigLocationForNamespace(namespace) |
| Kubernetes k8s = Kubernetes.create(delegate, kubeconfig, namespace) |
| |
| k8s.apply(common.makePathAbsolute("src/.test-infra/kubernetes/hadoop/LargeITCluster/hdfs-multi-datanode-cluster.yml")) |
| String hostName = "LOAD_BALANCER_IP" |
| k8s.loadBalancerIP("hadoop", hostName) |
| |
| Map additionalOptions = [ |
| runner : 'DataflowRunner', |
| project : 'apache-beam-testing', |
| tempRoot : 'gs://temp-storage-for-perf-tests', |
| hdfsConfiguration: /[{\\\"fs.defaultFS\\\":\\\"hdfs:$${hostName}:9000\\\",\\\"dfs.replication\\\":1}]/, |
| filenamePrefix : "hdfs://\$${hostName}:9000/TEXTIO_IT_" |
| ] |
| |
| Map allPipelineOptions = testJob.pipelineOptions << additionalOptions |
| String runner = "dataflow" |
| String filesystem = "hdfs" |
| String testTask = ":sdks:java:io:file-based-io-tests:integrationTest" |
| |
| steps { |
| gradle { |
| rootBuildScriptDir(common.checkoutDir) |
| common.setGradleSwitches(delegate) |
| switches("--info") |
| switches("-DintegrationTestPipelineOptions=\'${common.joinPipelineOptions(allPipelineOptions)}\'") |
| switches("-Dfilesystem=\'${filesystem}\'") |
| switches("-DintegrationTestRunner=\'${runner}\'") |
| tasks("${testTask} --tests ${testJob.test}") |
| } |
| } |
| } |
| } |
