diff --git a/.gitignore b/.gitignore
index 0c3444f..7e20ebd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -60,14 +60,6 @@
 ansible/roles/nginx/files/*.csr
 ansible/roles/nginx/files/*cert.pem
 
-# .zip files must be explicited whitelisted
+# Generated by tests:buildArtifacts
+.built
 *.zip
-!tests/dat/actions/blackbox.zip
-!tests/dat/actions/helloSwift.zip
-!tests/dat/actions/python.zip
-!tests/dat/actions/python2_virtualenv.zip
-!tests/dat/actions/python3_virtualenv.zip
-!tests/dat/actions/python_virtualenv_dir.zip
-!tests/dat/actions/python_virtualenv_name.zip
-!tests/dat/actions/zippedaction.zip
-!tests/dat/without_dir_entries.zip
diff --git a/tests/build.gradle b/tests/build.gradle
index 1ae5a4b..c1f63e0 100644
--- a/tests/build.gradle
+++ b/tests/build.gradle
@@ -42,3 +42,14 @@
 tasks.withType(ScalaCompile) {
     scalaCompileOptions.additionalParameters = gradle.scala.compileFlags
 }
+
+task buildArtifacts(type:Exec) {
+    workingDir 'src/test/resources'
+    commandLine './build.sh'
+}
+
+tasks.withType(Test) {
+    dependsOn buildArtifacts
+}
+
+testClasses.dependsOn(buildArtifacts)
diff --git a/tests/dat/without_dir_entries.zip b/tests/dat/without_dir_entries.zip
deleted file mode 100644
index 224a5f9..0000000
--- a/tests/dat/without_dir_entries.zip
+++ /dev/null
Binary files differ
diff --git a/tests/src/test/resources/build.sh b/tests/src/test/resources/build.sh
new file mode 100755
index 0000000..b74ea07
--- /dev/null
+++ b/tests/src/test/resources/build.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+if [ -f ".built" ]; then
+  echo "Test zip artifacts already built, skipping"
+  exit 0
+fi
+
+(cd without_dir_entries && zip ../without_dir_entries.zip -r -D .)
+
+touch .built
diff --git a/tests/src/test/resources/without_dir_entries/bundle/bundler/setup.rb b/tests/src/test/resources/without_dir_entries/bundle/bundler/setup.rb
new file mode 100644
index 0000000..202c2a2
--- /dev/null
+++ b/tests/src/test/resources/without_dir_entries/bundle/bundler/setup.rb
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'rbconfig'
+# ruby 1.8.7 doesn't define RUBY_ENGINE
+ruby_engine = defined?(RUBY_ENGINE) ? RUBY_ENGINE : 'ruby'
+ruby_version = RbConfig::CONFIG["ruby_version"]
+path = File.expand_path('..', __FILE__)
+$:.unshift "#{path}/../#{ruby_engine}/#{ruby_version}/gems/betterlorem-0.1.2/lib"
+$:.unshift "#{path}/"
diff --git a/tests/src/test/resources/without_dir_entries/main.rb b/tests/src/test/resources/without_dir_entries/main.rb
new file mode 100644
index 0000000..42f5276
--- /dev/null
+++ b/tests/src/test/resources/without_dir_entries/main.rb
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+def main(params = {})
+  name = params["name"] || "stranger"
+  greeting = "Hello #{name}!"
+  { greeting: greeting }
+end
diff --git a/tests/src/test/scala/actionContainers/Ruby25ActionContainerTests.scala b/tests/src/test/scala/actionContainers/Ruby25ActionContainerTests.scala
index c93d5a4..4e5839b 100644
--- a/tests/src/test/scala/actionContainers/Ruby25ActionContainerTests.scala
+++ b/tests/src/test/scala/actionContainers/Ruby25ActionContainerTests.scala
@@ -333,7 +333,7 @@
   }
 
   it should "support zip-encoded packages without directory entries" in {
-    val path = FileSystems.getDefault().getPath("dat", "without_dir_entries.zip");
+    val path = FileSystems.getDefault().getPath("src", "test", "resources", "without_dir_entries.zip");
     val code = ResourceHelpers.readAsBase64(path)
 
     val (out, err) = withRuby25Container { c =>