Tighten container reclamation.

If the result of running an action in the container is not successful, delete the container. Makes this is true for both failed init and run.

Test for container reclamation on action error.

Move error messages to canonical Messages object.
diff --git a/tests/src/whisk/core/cli/test/WskBasicUsageTests.scala b/tests/src/whisk/core/cli/test/WskBasicUsageTests.scala
index 7948697..ac5414d 100644
--- a/tests/src/whisk/core/cli/test/WskBasicUsageTests.scala
+++ b/tests/src/whisk/core/cli/test/WskBasicUsageTests.scala
@@ -344,7 +344,7 @@
             withActivation(wsk.activation, wsk.action.invoke(name)) {
                 activation =>
                     val response = activation.response
-                    response.result.get.fields("error") shouldBe ActivationResponse.abnormalInitialization
+                    response.result.get.fields("error") shouldBe Messages.abnormalInitialization.toJson
                     response.status shouldBe ActivationResponse.messageForCode(ActivationResponse.ContainerError)
             }
     }
@@ -363,7 +363,7 @@
             withActivation(wsk.activation, wsk.action.invoke(name)) {
                 activation =>
                     val response = activation.response
-                    response.result.get.fields("error") shouldBe ActivationResponse.timedoutActivation(3 seconds, true)
+                    response.result.get.fields("error") shouldBe Messages.timedoutActivation(3 seconds, true).toJson
                     response.status shouldBe ActivationResponse.messageForCode(ActivationResponse.ApplicationError)
             }
     }
@@ -378,7 +378,7 @@
             withActivation(wsk.activation, wsk.action.invoke(name)) {
                 activation =>
                     val response = activation.response
-                    response.result.get.fields("error") shouldBe ActivationResponse.abnormalRun
+                    response.result.get.fields("error") shouldBe Messages.abnormalRun.toJson
                     response.status shouldBe ActivationResponse.messageForCode(ActivationResponse.ContainerError)
             }
     }
@@ -513,6 +513,37 @@
             }
     }
 
+    it should s"invoke an action twice, where the first times out but the second does not and should succeed" in withAssetCleaner(wskprops) {
+        // this test issues two activations: the first is forced to time out and not return a result by its deadline (ie it does not resolve
+        // its promise). The invoker should reclaim its container so that a second activation of the same action (which must happen within a
+        // short period of time (seconds, not minutes) is allocated a fresh container and hence runs as expected (vs. hitting in the container
+        // cache and reusing a bad container).
+        (wp, assetHelper) =>
+            val name = "timeout"
+            assetHelper.withCleaner(wsk.action, name) {
+                (action, _) => action.create(name, Some(TestUtils.getTestActionFilename("helloDeadline.js")), timeout = Some(3 seconds))
+            }
+
+            val start = Instant.now(Clock.systemUTC()).toEpochMilli
+            val hungRun = wsk.action.invoke(name, Map("forceHang" -> true.toJson))
+            withActivation(wsk.activation, hungRun) {
+                activation =>
+                    // the first action must fail with a timeout error
+                    activation.response.status shouldBe ActivationResponse.messageForCode(ActivationResponse.ApplicationError)
+                    activation.response.result shouldBe Some(JsObject("error" -> Messages.timedoutActivation(3 seconds, false).toJson))
+            }
+
+            // run the action again, this time without forcing it to timeout
+            // it should succeed because it ran in a fresh container
+            val goodRun = wsk.action.invoke(name, Map("forceHang" -> false.toJson))
+            withActivation(wsk.activation, goodRun) {
+                activation =>
+                    // the first action must fail with a timeout error
+                    activation.response.status shouldBe "success"
+                    activation.response.result shouldBe Some(JsObject("timedout" -> true.toJson))
+            }
+    }
+
     behavior of "Wsk packages"
 
     it should "create, and delete a package" in {
@@ -713,38 +744,38 @@
         val badpath = "badpath"
 
         var rr = wsk.cli(Seq("api-experimental", "create", "/basepath", badpath, "GET", "action", "--auth", wskprops.authKey) ++ wskprops.overrides, expectedExitCode = ANY_ERROR_EXIT)
-        rr.stderr should include (s"'${badpath}' must begin with '/'")
+        rr.stderr should include(s"'${badpath}' must begin with '/'")
 
         rr = wsk.cli(Seq("api-experimental", "delete", "/basepath", badpath, "GET", "--auth", wskprops.authKey) ++ wskprops.overrides, expectedExitCode = ANY_ERROR_EXIT)
-        rr.stderr should include (s"'${badpath}' must begin with '/'")
+        rr.stderr should include(s"'${badpath}' must begin with '/'")
 
         rr = wsk.cli(Seq("api-experimental", "list", "/basepath", badpath, "GET", "--auth", wskprops.authKey) ++ wskprops.overrides, expectedExitCode = ANY_ERROR_EXIT)
-        rr.stderr should include (s"'${badpath}' must begin with '/'")
+        rr.stderr should include(s"'${badpath}' must begin with '/'")
     }
 
     it should "reject an api commands with an invalid verb parameter" in {
         val badverb = "badverb"
 
         var rr = wsk.cli(Seq("api-experimental", "create", "/basepath", "/path", badverb, "action", "--auth", wskprops.authKey) ++ wskprops.overrides, expectedExitCode = ANY_ERROR_EXIT)
-        rr.stderr should include (s"'${badverb}' is not a valid API verb.  Valid values are:")
+        rr.stderr should include(s"'${badverb}' is not a valid API verb.  Valid values are:")
 
         rr = wsk.cli(Seq("api-experimental", "delete", "/basepath", "/path", badverb, "--auth", wskprops.authKey) ++ wskprops.overrides, expectedExitCode = ANY_ERROR_EXIT)
-        rr.stderr should include (s"'${badverb}' is not a valid API verb.  Valid values are:")
+        rr.stderr should include(s"'${badverb}' is not a valid API verb.  Valid values are:")
 
         rr = wsk.cli(Seq("api-experimental", "list", "/basepath", "/path", badverb, "--auth", wskprops.authKey) ++ wskprops.overrides, expectedExitCode = ANY_ERROR_EXIT)
-        rr.stderr should include (s"'${badverb}' is not a valid API verb.  Valid values are:")
+        rr.stderr should include(s"'${badverb}' is not a valid API verb.  Valid values are:")
     }
 
     it should "reject an api create command with an API name argument and an API name option" in {
         val apiName = "An API Name"
         val rr = wsk.cli(Seq("api-experimental", "create", apiName, "/path", "GET", "action", "-n", apiName, "--auth", wskprops.authKey) ++ wskprops.overrides, expectedExitCode = ANY_ERROR_EXIT)
-        rr.stderr should include (s"An API name can only be specified once.")
+        rr.stderr should include(s"An API name can only be specified once.")
     }
 
     it should "reject an api create command that specifies a nonexistent configuration file" in {
         val configfile = "/nonexistent/file"
         val rr = wsk.cli(Seq("api-experimental", "create", "-c", configfile, "--auth", wskprops.authKey) ++ wskprops.overrides, expectedExitCode = ANY_ERROR_EXIT)
-        rr.stderr should include (s"Error reading swagger file '${configfile}':")
+        rr.stderr should include(s"Error reading swagger file '${configfile}':")
     }
 
     it should "reject an api create command specifying a non-JSON configuration file" in {
@@ -757,7 +788,7 @@
         bw.close()
 
         val rr = wsk.cli(Seq("api-experimental", "create", "-c", filename, "--auth", wskprops.authKey) ++ wskprops.overrides, expectedExitCode = ANY_ERROR_EXIT)
-        rr.stderr should include (s"Error parsing swagger file '${filename}':")
+        rr.stderr should include(s"Error parsing swagger file '${filename}':")
     }
 
     it should "reject an api create command specifying a non-swagger JSON configuration file" in {
@@ -778,12 +809,11 @@
                     |       "get":{}
                     |     }
                     |   }
-                    |}""".stripMargin
-        )
+                    |}""".stripMargin)
         bw.close()
 
         val rr = wsk.cli(Seq("api-experimental", "create", "-c", filename, "--auth", wskprops.authKey) ++ wskprops.overrides, expectedExitCode = ANY_ERROR_EXIT)
-        rr.stderr should include (s"Swagger file is invalid (missing basePath, info, paths, or swagger fields")
+        rr.stderr should include(s"Swagger file is invalid (missing basePath, info, paths, or swagger fields")
     }
 
     behavior of "Wsk entity list formatting"
diff --git a/tests/src/whisk/core/dispatcher/test/ActivationResponseTests.scala b/tests/src/whisk/core/dispatcher/test/ActivationResponseTests.scala
index c633447..66777b2 100644
--- a/tests/src/whisk/core/dispatcher/test/ActivationResponseTests.scala
+++ b/tests/src/whisk/core/dispatcher/test/ActivationResponseTests.scala
@@ -27,6 +27,7 @@
 import spray.json.pimpString
 import whisk.common.Logging
 import whisk.core.entity.ActivationResponse._
+import whisk.http.Messages._
 
 @RunWith(classOf[JUnitRunner])
 class ActivationResponseTests extends FlatSpec with Matchers {
@@ -38,14 +39,14 @@
     it should "interpret failed init that does not response" in {
         val ar = processInitResponseContent(None, logger)
         ar.statusCode shouldBe ContainerError
-        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe abnormalInitialization
+        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe abnormalInitialization.toJson
     }
 
     it should "interpret failed init that responds with null string" in {
         val response = Some(500, null)
         val ar = processInitResponseContent(response, logger)
         ar.statusCode shouldBe ContainerError
-        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidInitResponse(response.get._2)
+        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidInitResponse(response.get._2).toJson
         ar.result.get.toString should not include regex("null")
     }
 
@@ -53,7 +54,7 @@
         val response = Some(500, "")
         val ar = processInitResponseContent(response, logger)
         ar.statusCode shouldBe ContainerError
-        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidInitResponse(response.get._2)
+        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidInitResponse(response.get._2).toJson
         ar.result.get.asJsObject.fields(ERROR_FIELD).toString.endsWith(".\"") shouldBe true
     }
 
@@ -61,7 +62,7 @@
         val response = Some(500, "true")
         val ar = processInitResponseContent(response, logger)
         ar.statusCode shouldBe ContainerError
-        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidInitResponse(response.get._2)
+        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidInitResponse(response.get._2).toJson
         ar.result.get.toString should include(response.get._2)
     }
 
@@ -69,7 +70,7 @@
         val response = Some(500, Vector(1).toJson.compactPrint)
         val ar = processInitResponseContent(response, logger)
         ar.statusCode shouldBe ContainerError
-        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidInitResponse(response.get._2)
+        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidInitResponse(response.get._2).toJson
         ar.result.get.toString should include(response.get._2)
     }
 
@@ -84,7 +85,7 @@
         val response = Some(500, Map("foobar" -> "baz").toJson.compactPrint)
         val ar = processInitResponseContent(response, logger)
         ar.statusCode shouldBe ContainerError
-        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidInitResponse(response.get._2)
+        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidInitResponse(response.get._2).toJson
         ar.result.get.toString should include("baz")
     }
 
@@ -98,14 +99,14 @@
     it should "interpret failed run that does not response" in {
         val ar = processRunResponseContent(None, logger)
         ar.statusCode shouldBe ContainerError
-        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe abnormalRun
+        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe abnormalRun.toJson
     }
 
     it should "interpret failed run that responds with null string" in {
         val response = Some(500, null)
         val ar = processRunResponseContent(response, logger)
         ar.statusCode shouldBe ContainerError
-        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidRunResponse(response.get._2)
+        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidRunResponse(response.get._2).toJson
         ar.result.get.toString should not include regex("null")
     }
 
@@ -113,7 +114,7 @@
         val response = Some(500, "")
         val ar = processRunResponseContent(response, logger)
         ar.statusCode shouldBe ContainerError
-        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidRunResponse(response.get._2)
+        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidRunResponse(response.get._2).toJson
         ar.result.get.asJsObject.fields(ERROR_FIELD).toString.endsWith(".\"") shouldBe true
     }
 
@@ -121,7 +122,7 @@
         val response = Some(500, "true")
         val ar = processRunResponseContent(response, logger)
         ar.statusCode shouldBe ContainerError
-        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidRunResponse(response.get._2)
+        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidRunResponse(response.get._2).toJson
         ar.result.get.toString should include(response.get._2)
     }
 
@@ -129,7 +130,7 @@
         val response = Some(500, Vector(1).toJson.compactPrint)
         val ar = processRunResponseContent(response, logger)
         ar.statusCode shouldBe ContainerError
-        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidRunResponse(response.get._2)
+        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidRunResponse(response.get._2).toJson
         ar.result.get.toString should include(response.get._2)
     }
 
@@ -144,7 +145,7 @@
         val response = Some(500, Map("foobar" -> "baz").toJson.compactPrint)
         val ar = processRunResponseContent(response, logger)
         ar.statusCode shouldBe ContainerError
-        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidRunResponse(response.get._2)
+        ar.result.get.asJsObject.fields(ERROR_FIELD) shouldBe invalidRunResponse(response.get._2).toJson
         ar.result.get.toString should include("baz")
     }
 
diff --git a/tests/src/whisk/core/limits/ActionLimitsTests.scala b/tests/src/whisk/core/limits/ActionLimitsTests.scala
index 940f670..9648c71 100644
--- a/tests/src/whisk/core/limits/ActionLimitsTests.scala
+++ b/tests/src/whisk/core/limits/ActionLimitsTests.scala
@@ -17,6 +17,7 @@
 package whisk.core.limits
 
 import java.io.File
+import java.io.PrintWriter
 
 import scala.concurrent.duration.DurationInt
 import scala.language.postfixOps
@@ -31,18 +32,14 @@
 import common.Wsk
 import common.WskProps
 import common.WskTestHelpers
-import spray.json.DefaultJsonProtocol._
-import spray.json.pimpAny
-import java.io.PrintWriter
-import whisk.core.entity.Exec
-import spray.json.DefaultJsonProtocol.LongJsonFormat
-import common.TestUtils
-import whisk.core.entity.size.SizeInt
-import whisk.core.entity.size.SizeString
 import spray.json._
 import spray.json.DefaultJsonProtocol._
-import whisk.core.entity.ActivationResponse
+import spray.json.pimpAny
+import whisk.core.entity.Exec
 import whisk.core.entity.LogLimit
+import whisk.core.entity.size.SizeInt
+import whisk.core.entity.size.SizeString
+import whisk.http.Messages
 
 @RunWith(classOf[JUnitRunner])
 class ActionLimitsTests extends TestHelpers with WskTestHelpers {
@@ -72,7 +69,7 @@
             val run = wsk.action.invoke(name, Map("payload" -> allowedActionDuration.plus(1 second).toMillis.toJson))
             withActivation(wsk.activation, run) {
                 _.response.result.get.fields("error") shouldBe {
-                    ActivationResponse.timedoutActivation(allowedActionDuration, false)
+                    Messages.timedoutActivation(allowedActionDuration, false).toJson
                 }
             }
     }