Cleaning up benchmark output directory
Including the name of the benchmark in the output folder hierarchy.
Fixing all tests to use a temporary folder for the output directory that
is removed after running.
Failing the tests if the output folder is already populated.
diff --git a/geode-benchmarks/src/main/java/org/apache/geode/benchmark/tests/PartitionedPutBenchmark.java b/geode-benchmarks/src/main/java/org/apache/geode/benchmark/tests/PartitionedPutBenchmark.java
index cd13c33..a4c4b76 100644
--- a/geode-benchmarks/src/main/java/org/apache/geode/benchmark/tests/PartitionedPutBenchmark.java
+++ b/geode-benchmarks/src/main/java/org/apache/geode/benchmark/tests/PartitionedPutBenchmark.java
@@ -37,6 +37,7 @@
int locatorPort = 10334;
+ config.name(PartitionedPutBenchmark.class.getCanonicalName());
config.warmupSeconds(5);
config.durationSeconds(30);
config.role("locator", 1);
diff --git a/geode-benchmarks/src/test/java/org/apache/geode/benchmark/PartitionedPutBenchmarkTest.java b/geode-benchmarks/src/test/java/org/apache/geode/benchmark/PartitionedPutBenchmarkTest.java
index 377c464..4822dab 100644
--- a/geode-benchmarks/src/test/java/org/apache/geode/benchmark/PartitionedPutBenchmarkTest.java
+++ b/geode-benchmarks/src/test/java/org/apache/geode/benchmark/PartitionedPutBenchmarkTest.java
@@ -17,20 +17,23 @@
package org.apache.geode.benchmark;
-import org.junit.Test;
+import java.io.File;
-import org.apache.geode.benchmark.tasks.PutTask;
-import org.apache.geode.benchmark.tasks.StartClient;
-import org.apache.geode.benchmark.tasks.StartLocator;
-import org.apache.geode.benchmark.tasks.StartServer;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
import org.apache.geode.benchmark.tests.PartitionedPutBenchmark;
-import org.apache.geode.perftest.TestConfig;
import org.apache.geode.perftest.TestRunners;
public class PartitionedPutBenchmarkTest {
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
@Test
public void benchmarkRunsSuccessfully() throws Exception {
- TestRunners.minimalRunner().runTest(new PartitionedPutBenchmark()::configure);
+ TestRunners.minimalRunner(folder.newFolder())
+ .runTest(new PartitionedPutBenchmark()::configure);
}
}
diff --git a/harness/src/main/java/org/apache/geode/perftest/TestConfig.java b/harness/src/main/java/org/apache/geode/perftest/TestConfig.java
index 985ad1e..e3acb06 100644
--- a/harness/src/main/java/org/apache/geode/perftest/TestConfig.java
+++ b/harness/src/main/java/org/apache/geode/perftest/TestConfig.java
@@ -39,6 +39,7 @@
private List<TestStep> before = new ArrayList<>();
private List<TestStep> workload = new ArrayList<>();
private List<TestStep> after = new ArrayList<>();
+ private String name;
/**
* Define a role for the test.
@@ -77,7 +78,7 @@
* @param roles The roles to run the workload on
*/
public void workload(BenchmarkDriver benchmark, String ... roles) {
- workload.add(new TestStep(new YardstickTask(benchmark, workloadConfig), roles));
+ workload.add(new TestStep(new YardstickTask(benchmark, workloadConfig, "output"), roles));
}
@@ -108,6 +109,14 @@
workloadConfig.warmupSeconds(warmupSeconds);
}
+ /**
+ * Set the name of this benchmark. This name must be unique within a benchmarking run. Comparisons
+ * between runs will use this name to identify the same benchmark in both runs.
+ */
+ public void name(String name) {
+ this.name = name;
+ }
+
public long getDurationSeconds() {
return workloadConfig.getDurationSeconds();
}
@@ -132,6 +141,10 @@
return after;
}
+ public String getName() {
+ return name;
+ }
+
/**
* Return the total number of JVMs required to run this test
* @return
diff --git a/harness/src/main/java/org/apache/geode/perftest/TestRunners.java b/harness/src/main/java/org/apache/geode/perftest/TestRunners.java
index fc6282a..d3dc6b5 100644
--- a/harness/src/main/java/org/apache/geode/perftest/TestRunners.java
+++ b/harness/src/main/java/org/apache/geode/perftest/TestRunners.java
@@ -17,7 +17,8 @@
package org.apache.geode.perftest;
-import org.apache.geode.perftest.infrastructure.InfrastructureFactory;
+import java.io.File;
+
import org.apache.geode.perftest.infrastructure.local.LocalInfrastructureFactory;
import org.apache.geode.perftest.infrastructure.ssh.SshInfrastructureFactory;
import org.apache.geode.perftest.jvms.RemoteJVMFactory;
@@ -40,7 +41,8 @@
public static TestRunner defaultRunner(String username, String ... hosts) {
- return new DefaultTestRunner(new SshInfrastructureFactory(username, hosts), new RemoteJVMFactory());
+ return new DefaultTestRunner(new SshInfrastructureFactory(username, hosts), new RemoteJVMFactory(),
+ new File("output"));
}
/**
* The default runner, which gets a list of hosts to run on from the
@@ -65,9 +67,11 @@
/**
* A test runner that runs the test with the minimal tuning - only
* 1 second duration on local infrastructure.
+ * @param outputDir
*/
- public static TestRunner minimalRunner() {
- return new DefaultTestRunner(new LocalInfrastructureFactory(), new RemoteJVMFactory()) {
+ public static TestRunner minimalRunner(final File outputDir) {
+ return new DefaultTestRunner(new LocalInfrastructureFactory(), new RemoteJVMFactory(),
+ outputDir) {
@Override
public void runTest(TestConfig config) throws Exception {
config.warmupSeconds(0);
diff --git a/harness/src/main/java/org/apache/geode/perftest/jvms/rmi/ChildJVM.java b/harness/src/main/java/org/apache/geode/perftest/jvms/rmi/ChildJVM.java
index 478902b..7aa9b8a 100644
--- a/harness/src/main/java/org/apache/geode/perftest/jvms/rmi/ChildJVM.java
+++ b/harness/src/main/java/org/apache/geode/perftest/jvms/rmi/ChildJVM.java
@@ -33,15 +33,17 @@
private final RMI rmi;
private final SystemInterface system;
private final int pingTime;
+ private final File outputDir;
- public ChildJVM(RMI rmi, SystemInterface system, int pingTime) {
+ public ChildJVM(RMI rmi, SystemInterface system, int pingTime, File outputDir) {
this.rmi = rmi;
this.system = system;
this.pingTime = pingTime;
+ this.outputDir = outputDir;
}
public static void main(String[] args) {
- new ChildJVM(new RMI(), new SystemInterface(), 1000).run();
+ new ChildJVM(new RMI(), new SystemInterface(), 1000, new File("output")).run();
}
void run() {
@@ -49,7 +51,6 @@
String RMI_HOST = system.getProperty(RemoteJVMFactory.RMI_HOST);
String RMI_PORT = system.getProperty(RemoteJVMFactory.RMI_PORT_PROPERTY);
int id = system.getInteger(RemoteJVMFactory.JVM_ID);
- File outputDir = new File("output");
outputDir.mkdirs();
PrintStream out = new PrintStream(new File(outputDir, "ChildJVM-" + id + ".txt"));
system.setOut(out);
diff --git a/harness/src/main/java/org/apache/geode/perftest/runner/DefaultTestRunner.java b/harness/src/main/java/org/apache/geode/perftest/runner/DefaultTestRunner.java
index 54774f5..6b490ed 100644
--- a/harness/src/main/java/org/apache/geode/perftest/runner/DefaultTestRunner.java
+++ b/harness/src/main/java/org/apache/geode/perftest/runner/DefaultTestRunner.java
@@ -18,10 +18,8 @@
package org.apache.geode.perftest.runner;
import java.io.File;
-import java.io.IOException;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,10 +46,13 @@
private final InfrastructureFactory infrastructureFactory;
private final RemoteJVMFactory remoteJvmFactory;
+ private File outputDir;
- public DefaultTestRunner(InfrastructureFactory infrastructureFactory, RemoteJVMFactory remoteJvmFactory) {
+ public DefaultTestRunner(InfrastructureFactory infrastructureFactory,
+ RemoteJVMFactory remoteJvmFactory, File outputDir) {
this.infrastructureFactory = infrastructureFactory;
this.remoteJvmFactory = remoteJvmFactory;
+ this.outputDir = outputDir;
}
@Override
@@ -65,6 +66,15 @@
throws Exception {
int nodes = config.getTotalJVMs();
+ if(config.getName() == null) {
+ throw new IllegalStateException("Benchmark must have a name.");
+ }
+ File benchmarkOutput = new File(outputDir, config.getName());
+ if(benchmarkOutput.exists()) {
+ throw new IllegalStateException("Benchmark output directory already exists: " + benchmarkOutput.getPath());
+ }
+
+
try (Infrastructure infra = infrastructureFactory.create(nodes)){
Map<String, Integer> roles = config.getRoles();
@@ -82,11 +92,15 @@
runTasks(config.getAfter(), remoteJVMs);
logger.info("Copying results...");
- File outputDir = new File("output");
+
int nodeId = 0;
+
+
+ benchmarkOutput.mkdirs();
+
for (Infrastructure.Node node : infra.getNodes()) {
String role = remoteJVMs.getRole(node);
- infra.copyFromNode(node, "output", new File(outputDir, role + nodeId++));
+ infra.copyFromNode(node, "output", new File(benchmarkOutput, role + nodeId++));
}
}
}
diff --git a/harness/src/main/java/org/apache/geode/perftest/yardstick/YardstickTask.java b/harness/src/main/java/org/apache/geode/perftest/yardstick/YardstickTask.java
index 1e25056..80f4c1a 100644
--- a/harness/src/main/java/org/apache/geode/perftest/yardstick/YardstickTask.java
+++ b/harness/src/main/java/org/apache/geode/perftest/yardstick/YardstickTask.java
@@ -42,11 +42,13 @@
*/
public class YardstickTask implements Task {
private final BenchmarkDriver benchmark;
+ private final String outputDir;
private WorkloadConfig workloadConfig;
- public YardstickTask(BenchmarkDriver benchmark, WorkloadConfig workloadConfig) {
+ public YardstickTask(BenchmarkDriver benchmark, WorkloadConfig workloadConfig, String outputDir) {
this.benchmark = benchmark;
this.workloadConfig = workloadConfig;
+ this.outputDir = outputDir;
}
@Override
@@ -75,7 +77,7 @@
@Override
public String outputFolder() {
- return "output";
+ return outputDir;
}
};
cfg.output(System.out);
diff --git a/harness/src/test/java/org/apache/geode/perftest/TestRunnerIntegrationTest.java b/harness/src/test/java/org/apache/geode/perftest/TestRunnerIntegrationTest.java
index c011da6..fac20a9 100644
--- a/harness/src/test/java/org/apache/geode/perftest/TestRunnerIntegrationTest.java
+++ b/harness/src/test/java/org/apache/geode/perftest/TestRunnerIntegrationTest.java
@@ -17,23 +17,78 @@
package org.apache.geode.perftest;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.function.BiPredicate;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import org.apache.geode.perftest.benchmarks.EmptyBenchmark;
import org.apache.geode.perftest.infrastructure.local.LocalInfrastructureFactory;
import org.apache.geode.perftest.jvms.RemoteJVMFactory;
import org.apache.geode.perftest.runner.DefaultTestRunner;
+import org.apache.geode.perftest.yardstick.analysis.YardstickThroughputSensorParser;
public class TestRunnerIntegrationTest {
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+ private TestRunner runner;
+ private File outputDir;
+ public static final String SAMPLE_BENCHMARK = "SampleBenchmark";
+
+ @Before
+ public void setup() throws IOException {
+ outputDir = temporaryFolder.newFolder();
+ runner = new DefaultTestRunner(new LocalInfrastructureFactory(), new RemoteJVMFactory(),
+ outputDir);
+ }
@Test
public void runsBeforeWorkload() throws Exception {
- TestRunner runner = new DefaultTestRunner(new LocalInfrastructureFactory(), new RemoteJVMFactory());
-
runner.runTest(testConfig -> {
+ testConfig.name(SAMPLE_BENCHMARK);
testConfig.role("all", 1);
testConfig.before(context -> System.out.println("hello"), "all");
+ });
+ }
+ @Test
+ public void generatesOutputDirectoryPerBenchmark() throws Exception {
+
+ runner.runTest(testConfig -> {
+ testConfig.name(SAMPLE_BENCHMARK);
+ testConfig.role("all", 1);
+ testConfig.workload(new EmptyBenchmark(), "all");
});
+ File expectedBenchmarkDir = new File(outputDir, SAMPLE_BENCHMARK);
+ assertTrue(expectedBenchmarkDir.exists());
+
+ //Node directory name is the role + a number
+ File expectedNodeDir = new File(expectedBenchmarkDir, "all0");
+ assertTrue(expectedNodeDir.exists());
+
+ //We expect the node directory to have benchmark results
+ Stream<Path>
+ outputFiles = Files.walk(expectedNodeDir.toPath())
+ .filter(nameMatches(YardstickThroughputSensorParser.sensorOutputFile));
+
+ assertEquals(1, outputFiles.count());
+ }
+
+ private Predicate<Path> nameMatches(String sensorOutputFile) {
+ return path -> path.toString().contains(sensorOutputFile);
}
}
diff --git a/harness/src/test/java/org/apache/geode/perftest/TestRunnerJUnitTest.java b/harness/src/test/java/org/apache/geode/perftest/TestRunnerJUnitTest.java
index 0ae3fee..c3a5e81 100644
--- a/harness/src/test/java/org/apache/geode/perftest/TestRunnerJUnitTest.java
+++ b/harness/src/test/java/org/apache/geode/perftest/TestRunnerJUnitTest.java
@@ -25,7 +25,11 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.io.File;
+
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.mockito.InOrder;
import org.apache.geode.perftest.infrastructure.InfrastructureFactory;
@@ -36,6 +40,9 @@
public class TestRunnerJUnitTest {
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
@Test
public void testRunnerRunsBeforeAndAfterTasks() throws Exception {
@@ -48,12 +55,14 @@
RemoteJVMs remoteJVMs = mock(RemoteJVMs.class);
when(remoteJvmFactory.launch(eq(infrastructure), any())).thenReturn(remoteJVMs);
- TestRunner runner = new DefaultTestRunner(infrastructureFactory, remoteJvmFactory);
+ TestRunner runner = new DefaultTestRunner(infrastructureFactory, remoteJvmFactory,
+ folder.newFolder());
Task before = mock(Task.class);
Task after = mock(Task.class);
PerformanceTest test = config -> {
+ config.name("SampleBenchmark");
config.role("before", 1);
config.role("workload", 1);
config.role("after", 1);
diff --git a/harness/src/test/java/org/apache/geode/perftest/benchmarks/EmptyBenchmark.java b/harness/src/test/java/org/apache/geode/perftest/benchmarks/EmptyBenchmark.java
new file mode 100644
index 0000000..02e0626
--- /dev/null
+++ b/harness/src/test/java/org/apache/geode/perftest/benchmarks/EmptyBenchmark.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.geode.perftest.benchmarks;
+
+import java.io.Serializable;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.yardstickframework.BenchmarkDriverAdapter;
+
+public class EmptyBenchmark extends BenchmarkDriverAdapter implements Serializable {
+ private AtomicInteger invocations = new AtomicInteger();
+
+ @Override
+ public boolean test(Map<Object, Object> ctx) throws Exception {
+ invocations.incrementAndGet();
+ return true;
+ }
+
+ public int getInvocations() {
+ return this.invocations.get();
+ }
+
+ @Override
+ public void onException(Throwable e) {
+ e.printStackTrace();
+ }
+}
diff --git a/harness/src/test/java/org/apache/geode/perftest/jvms/rmi/ChildJVMTest.java b/harness/src/test/java/org/apache/geode/perftest/jvms/rmi/ChildJVMTest.java
index 146ab61..3fa21ca 100644
--- a/harness/src/test/java/org/apache/geode/perftest/jvms/rmi/ChildJVMTest.java
+++ b/harness/src/test/java/org/apache/geode/perftest/jvms/rmi/ChildJVMTest.java
@@ -24,12 +24,16 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.io.File;
+import java.io.IOException;
import java.net.MalformedURLException;
import java.rmi.NotBoundException;
import java.rmi.RemoteException;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.apache.geode.perftest.jdk.RMI;
import org.apache.geode.perftest.jdk.SystemInterface;
@@ -37,16 +41,18 @@
public class ChildJVMTest {
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
private RMI rmi;
private ChildJVM jvm;
private SystemInterface system;
private Controller controller;
@Before
- public void setUp() throws RemoteException, NotBoundException, MalformedURLException {
+ public void setUp() throws IOException, NotBoundException {
system = mock(SystemInterface.class);
rmi = mock(RMI.class);
- jvm = new ChildJVM(rmi, system, 1);
+ jvm = new ChildJVM(rmi, system, 1, temporaryFolder.newFolder());
controller = mock(Controller.class);
when(rmi.lookup(any())).thenReturn(controller);
diff --git a/harness/src/test/java/org/apache/geode/perftest/yardstick/YardstickTaskTest.java b/harness/src/test/java/org/apache/geode/perftest/yardstick/YardstickTaskTest.java
index 7d60d9b..e7351e7 100644
--- a/harness/src/test/java/org/apache/geode/perftest/yardstick/YardstickTaskTest.java
+++ b/harness/src/test/java/org/apache/geode/perftest/yardstick/YardstickTaskTest.java
@@ -17,31 +17,29 @@
package org.apache.geode.perftest.yardstick;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
-
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-
import org.junit.Assert;
+import org.junit.Rule;
import org.junit.Test;
-import org.yardstickframework.BenchmarkDriverAdapter;
+import org.junit.rules.TemporaryFolder;
import org.apache.geode.perftest.Task;
import org.apache.geode.perftest.WorkloadConfig;
+import org.apache.geode.perftest.benchmarks.EmptyBenchmark;
public class YardstickTaskTest {
+ @Rule
+ public final TemporaryFolder folder = new TemporaryFolder();
+
@Test
public void testExecuteBenchmark() throws Exception {
EmptyBenchmark benchmark = new EmptyBenchmark();
WorkloadConfig workloadConfig = new WorkloadConfig();
workloadConfig.threads(1);
- Task task = new YardstickTask(benchmark, workloadConfig);
+ Task task = new YardstickTask(benchmark, workloadConfig, folder.newFolder().getAbsolutePath());
task.run(null);
- Assert.assertTrue(1 <= benchmark.invocations.get());
+ Assert.assertTrue(1 <= benchmark.getInvocations());
//TODO -verify probes are shutdown
//TODO -verify benchmark is shutdown
@@ -49,18 +47,4 @@
}
- private static class EmptyBenchmark extends BenchmarkDriverAdapter {
- private AtomicInteger invocations = new AtomicInteger();
-
- @Override
- public boolean test(Map<Object, Object> ctx) throws Exception {
- invocations.incrementAndGet();
- return true;
- }
-
- @Override
- public void onException(Throwable e) {
- e.printStackTrace();
- }
- }
}
\ No newline at end of file
