improve rest output with errors, and reduce max depth

catches issues sooner if there is an object that refers to itself
diff --git a/core/src/main/java/org/apache/brooklyn/util/core/json/BrooklynObjectsJsonMapper.java b/core/src/main/java/org/apache/brooklyn/util/core/json/BrooklynObjectsJsonMapper.java
index 5b2837c..4d036ba 100644
--- a/core/src/main/java/org/apache/brooklyn/util/core/json/BrooklynObjectsJsonMapper.java
+++ b/core/src/main/java/org/apache/brooklyn/util/core/json/BrooklynObjectsJsonMapper.java
@@ -15,26 +15,24 @@
  */
 package org.apache.brooklyn.util.core.json;
 
+import java.io.IOException;
+
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.StreamWriteConstraints;
+import com.fasterxml.jackson.core.Version;
 import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.datatype.jsr310.JSR310Module;
+import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import org.apache.brooklyn.api.mgmt.ManagementContext;
 import org.apache.brooklyn.api.mgmt.classloading.BrooklynClassLoadingContext;
 import org.apache.brooklyn.core.resolve.jackson.CommonTypesSerialization;
 import org.apache.brooklyn.util.time.Duration;
-
-import com.fasterxml.jackson.core.Version;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.util.Date;
-
 public class BrooklynObjectsJsonMapper {
 
     private static final Logger LOG = LoggerFactory.getLogger(BrooklynObjectsJsonMapper.class);
@@ -46,6 +44,10 @@
         sp.setUnknownTypeSerializer(new ErrorAndToStringUnknownTypeSerializer());
 
         ObjectMapper mapper = new ObjectMapper();
+
+        // default of 1000 is much more than we need or want
+        mapper.getFactory().setStreamWriteConstraints(StreamWriteConstraints.builder().maxNestingDepth(100).build());
+
         mapper.setSerializerProvider(sp);
         mapper.setVisibility(new PossiblyStrictPreferringFieldsVisibilityChecker());
 
diff --git a/core/src/main/java/org/apache/brooklyn/util/core/json/ErrorAndToStringUnknownTypeSerializer.java b/core/src/main/java/org/apache/brooklyn/util/core/json/ErrorAndToStringUnknownTypeSerializer.java
index 3519882..5a540cf 100644
--- a/core/src/main/java/org/apache/brooklyn/util/core/json/ErrorAndToStringUnknownTypeSerializer.java
+++ b/core/src/main/java/org/apache/brooklyn/util/core/json/ErrorAndToStringUnknownTypeSerializer.java
@@ -22,20 +22,20 @@
 import java.io.NotSerializableException;
 import java.util.Collections;
 import java.util.Set;
-
 import javax.annotation.Nullable;
 
-import org.apache.brooklyn.util.collections.MutableSet;
-import org.apache.brooklyn.util.javalang.Reflections;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonStreamContext;
+import com.fasterxml.jackson.core.json.JsonWriteContext;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.SerializerProvider;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import com.fasterxml.jackson.databind.ser.impl.UnknownSerializer;
+import org.apache.brooklyn.util.collections.MutableSet;
+import org.apache.brooklyn.util.exceptions.Exceptions;
+import org.apache.brooklyn.util.javalang.Reflections;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * for non-json-serializable classes (quite a lot of them!) simply provide a sensible error message and a toString.
@@ -66,16 +66,27 @@
         if (WARNED_CLASSES.add(value.getClass().getCanonicalName())) {
             log.warn("Standard serialization not possible for "+value.getClass()+" ("+value+")", error);
         }
-        JsonStreamContext newCtxt = jgen.getOutputContext();
 
-        // very odd, but flush seems necessary when working with large objects; presumably a buffer which is allowed to clear itself?
+        // flush seems necessary when working with large objects; presumably a buffer which is allowed to clear itself?
         // without this, when serializing the large (1.5M) Server json object from BrooklynJacksonSerializerTest creates invalid json,
         // containing:  "foo":false,"{"error":true,...
         jgen.flush();
 
-        boolean createObject = !newCtxt.inObject() || newCtxt.getCurrentName()!=null;
+        // if nested very deeply, come out, because there will be errors writing deeper things
+        int closed = 0;
+        while (jgen.getOutputContext().getNestingDepth() > 30 && writeEndCurrentThing(jgen, closed++)) {}
+        if (jgen.getOutputContext().getNestingDepth() > 30) {
+            throw new IllegalStateException("Cannot recover from serialization object; nesting is too deep");
+        }
+
+        boolean createObject = !jgen.getOutputContext().inObject();
         if (createObject) {
+            // create if we're not in an object (ie in an array)
+            // or if we're in an object, but we've just written the field name
             jgen.writeStartObject();
+        } else {
+            // we might need to write a value, and then write the error fields next
+            writeErrorValueIfNeeded(jgen);
         }
 
         if (allowEmpty(value.getClass())) {
@@ -108,12 +119,36 @@
             jgen.writeEndObject();
         }
 
-        while (newCtxt!=null && !newCtxt.equals(ctxt)) {
-            if (jgen.getOutputContext().inArray()) { jgen.writeEndArray(); continue; }
-            if (jgen.getOutputContext().inObject()) { jgen.writeEndObject(); continue; }
-            break;
-        }
+        while (jgen.getOutputContext()!=null && !jgen.getOutputContext().equals(ctxt) && writeEndCurrentThing(jgen, closed+1)) {}
+    }
 
+    private static boolean writeEndCurrentThing(JsonGenerator jgen, int count) throws IOException {
+        if (jgen.getOutputContext().inArray()) {
+            jgen.writeEndArray();
+            return true;
+        }
+        if (jgen.getOutputContext().inObject()) {
+            if (count==0) writeErrorValueIfNeeded(jgen);
+            jgen.writeEndObject();
+            return true;
+        }
+        return false;
+    }
+
+    private static void writeErrorValueIfNeeded(JsonGenerator jgen) {
+        try {
+            // at count 0, we usually need to write a value
+            // (but there is no way to tell for sure; the internal status on JsonWriteContext is protected,
+            // and the jgen methods are closely coupled to their state; and any attempt to mutate will insert an extra colon etc)
+            if (jgen.getOutputContext().hasCurrentName()) {
+                // assume it wrote the name, but the output context might not have the correct state;
+                // have tried updated output context but it is mostly protected; instead just write this, usually good enough.
+                jgen.writeRaw("\"ERROR\"");
+            }
+        } catch (Exception e) {
+            Exceptions.propagateIfFatal(e);
+            // if we couldn't write, we're probably at a place where a field would be written, so just end
+        }
     }
 
     protected boolean allowEmpty(Class<? extends Object> clazz) {