src/java/org/apache/cassandra/service/RetryStrategy.java - cassandra - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 package org.apache.cassandra.service;

 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
 import java.util.function.DoubleSupplier;
 import java.util.function.LongBinaryOperator;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;

 import com.google.common.annotations.VisibleForTesting;

 import accord.utils.Invariants;
 import accord.utils.RandomSource;

 import org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory;
 import org.apache.cassandra.service.TimeoutStrategy.Wait;

 import static java.util.concurrent.TimeUnit.MICROSECONDS;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static org.apache.cassandra.service.TimeoutStrategy.parseInMicros;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;

 /**
  * <p>A strategy for making retry timing decisions for operations.
  * The strategy is defined by six factors: <ul>
  * <li> {@link #minMinMicros}
  * <li> {@link #maxMaxMicros}
  * <li> {@link #min}
  * <li> {@link #max}
  * <li> {@link #waitRandomizer}
  * <li> {@link #maxAttempts}
  * </ul>
  *
  * <p>The first two represent the absolute upper and lower bound times we are permitted to produce as constants</p>
  * <p>The next two represent time periods, and may be defined dynamically based on a simple calculation over: <ul>
  * <li> {@code pX()} recent experienced latency distribution for successful operations,
  *                 e.g. {@code p50(rw)} the maximum of read and write median latencies,
  *                      {@code p999(r)} the 99.9th percentile of read latencies
  * <li> {@code attempts} the number of failed attempts made by the operation so far
  * <li> {@code constant} a user provided floating point constant
  * </ul>
  *
  * <p>Their calculation may take any of these forms
  * <li> constant            {@code $constant$[mu]s}
  * <li> dynamic constant    {@code pX() * constant}
  * <li> dynamic linear      {@code pX() * constant * attempts}
  * <li> dynamic exponential {@code pX() * constant ^ attempts}
  * <li> Note: for dynamic exponential, attempts is subtracted by 1, such that times begin at {@code pX() * constant}.
  * e.g.
  * <li> {@code 10ms <= p50(rw)*0.66...p99(rw)}
  * <li> {@code 10ms <= p95(rw)*1.8^attempts <= 100ms}
  * <li> {@code 5ms <= p50(rw)*0.5}
  *
  * <p>These calculations are put together to construct a range from which we draw a random number.
  * The period we wait for {@code X} will be drawn so that {@code minMin <= min <= max <= maxMax}.
  *
  * <p>With the constraint that {@code max} must be {@code spread} greater than {@code min},
  * but no greater than its expression-defined maximum. {@code max} will be increased up until
  * this point, after which {@code min} will be decreased until this gap is imposed.
  *
  * <p>The {@link #waitRandomizer} property specifies the manner in which a random value is drawn from the range.
  * It is defined using one of the following specifiers:
  * <li> uniform
  * <li> exp($power$) or exponential($power$)
  * <li> qexp($power$) or qexponential($power$) or quantizedexponential($power$)
  *
  * The uniform specifier is self-explanatory, selecting all values in the range with equal probability.
  * The exponential specifier draws values towards the end of the range with higher probability, raising
  * a floating point number in the range [0..1.0) to the power provided, and translating the resulting value
  * to a uniform value in the range.
  * The quantized exponential specifier partitions the range into {@code attempts} buckets, then applies the pure
  * exponential approach to draw values from [0..attempts), before drawing a uniform value from the corresponding bucket
  */
 public class RetryStrategy implements WaitStrategy
 {
     private static final Pattern RANDOMIZER = Pattern.compile(
                 "uniform|exp(onential)?[(](?<exp>[0-9.]+)[)]|q(uantized)?exp(onential)?[(](?<qexp>[0-9.]+)[)]");

     static final Pattern PARSE = Pattern.compile(
           "(\\s*(?<minmin>0|[0-9]+[mu]?s)\\s*<=)?" +
                 "(\\s*(?<min>[^=]+)([(]?\\s*<=\\s*(?<maxmin>0|[0-9]+[mu]?s)\\s*[)]?)?\\s*[.]{3})?" +
                 "(\\s*(?<max>[^=]+))" +
                 "(\\s*<=\\s*(?<maxmax>0|[0-9]+[mu]?s))?");

     public static final WaitRandomizerFactory randomizers = new WaitRandomizerFactory(){};

     public interface WaitRandomizer
     {
         long wait(long min, long max, int attempts);
     }

     public static WaitRandomizerFactory randomizers(RandomSource random)
     {
         return new WaitRandomizerFactory()
         {
             @Override public LongBinaryOperator uniformLongSupplier() { return random::nextLong; }
             @Override public DoubleSupplier uniformDoubleSupplier() { return random::nextDouble; }
         };
     }

     public interface WaitRandomizerFactory
     {
         default LongBinaryOperator uniformLongSupplier() { return (min, max) -> ThreadLocalRandom.current().nextLong(min, max); } // DO NOT USE METHOD HANDLES (want to fetch afresh each time)
         default DoubleSupplier uniformDoubleSupplier() { return () -> ThreadLocalRandom.current().nextDouble(); }

         default WaitRandomizer uniform() { return new Uniform(uniformLongSupplier()); }
         default WaitRandomizer exponential(double power) { return new Exponential(uniformLongSupplier(), uniformDoubleSupplier(), power); }
         default WaitRandomizer quantizedExponential(double power) { return new QuantizedExponential(uniformLongSupplier(), uniformDoubleSupplier(), power); }

         class Uniform implements WaitRandomizer
         {
             final LongBinaryOperator uniformLong;

             public Uniform(LongBinaryOperator uniformLong)
             {
                 this.uniformLong = uniformLong;
             }

             @Override
             public long wait(long min, long max, int attempts)
             {
                 return uniformLong.applyAsLong(min, max);
             }
         }

         abstract class AbstractExponential implements WaitRandomizer
         {
             final LongBinaryOperator uniformLong;
             final DoubleSupplier uniformDouble;
             final double power;

             public AbstractExponential(LongBinaryOperator uniformLong, DoubleSupplier uniformDouble, double power)
             {
                 this.uniformLong = uniformLong;
                 this.uniformDouble = uniformDouble;
                 this.power = power;
             }
         }

         class Exponential extends AbstractExponential
         {
             public Exponential(LongBinaryOperator uniformLong, DoubleSupplier uniformDouble, double power)
             {
                 super(uniformLong, uniformDouble, power);
             }

             @Override
             public long wait(long min, long max, int attempts)
             {
                 if (attempts == 1)
                     return uniformLong.applyAsLong(min, max);

                 double p = uniformDouble.getAsDouble();
                 long delta = max - min;
                 delta *= Math.pow(p, power);
                 return max - delta;
             }
         }

         class QuantizedExponential extends AbstractExponential
         {
             public QuantizedExponential(LongBinaryOperator uniformLong, DoubleSupplier uniformDouble, double power)
             {
                 super(uniformLong, uniformDouble, power);
             }

             @Override
             public long wait(long min, long max, int attempts)
             {
                 long quanta = (max - min) / attempts;
                 if (attempts == 1 || quanta == 0)
                     return uniformLong.applyAsLong(min, max);

                 double p = uniformDouble.getAsDouble();
                 int base = (int) (attempts * Math.pow(p, power));
                 return max - ThreadLocalRandom.current().nextLong(quanta * base, quanta * (base + 1));
             }
         }
     }

     public final WaitRandomizer waitRandomizer;
     public final long minMinMicros, maxMinMicros, maxMaxMicros;
     public final @Nullable Wait min;
     public final @Nonnull Wait max;
     public final int maxAttempts;

     protected RetryStrategy(WaitRandomizer waitRandomizer, long minMinMicros, Wait min, long maxMinMicros, Wait max, long maxMaxMicros, int retries)
     {
         this.waitRandomizer = waitRandomizer;
         this.minMinMicros = minMinMicros;
         this.min = min;
         this.maxMinMicros = maxMinMicros;
         this.max = max;
         this.maxMaxMicros = maxMaxMicros;
         this.maxAttempts = retries == Integer.MAX_VALUE ? Integer.MAX_VALUE : retries + 1;
         Invariants.require(maxAttempts >= 1);
     }

     public long computeWaitUntil(int attempts)
     {
         long wait = computeWait(attempts, NANOSECONDS);
         if (wait < 0)
             return -1;
         return nanoTime() + wait;
     }

     public long computeWait(int attempt, TimeUnit units)
     {
         if (attempt > maxAttempts)
             return -1;

         long result;
         if (min == null)
         {
              result = max.getMicros(attempt);
         }
         else
         {
             long min = this.min.getMicros(attempt);
             if (min > maxMinMicros)
                 min = maxMinMicros;
             long max = this.max.getMicros(attempt);
             if (max > maxMaxMicros)
                 max = maxMaxMicros;
             result = min >= max ? min : waitRandomizer.wait(min, max, attempt);
         }

         if (result > maxMaxMicros) result = maxMaxMicros;
         if (result < minMinMicros) result = minMinMicros;
         return units.convert(result, MICROSECONDS);
     }

     public long computeMaxWait(int attempt, TimeUnit units)
     {
         return units.convert(max.getMaxMicros(attempt), MICROSECONDS);
     }

     public static RetryStrategy parse(String spec, LatencySourceFactory latencies)
     {
         return parse(spec, latencies, null);
     }

     public static RetryStrategy parse(String spec, LatencySourceFactory latencies, WaitRandomizer randomizer)
     {
         String original = spec;
         int retries = Integer.MAX_VALUE;
         int end = spec.length();
         {
             int next;
             while ((next = spec.lastIndexOf(',', end - 1)) >= 0)
             {
                 int mid = spec.indexOf('=', next + 1);
                 if (mid <= next || mid >= end)
                     throw new IllegalArgumentException("Invalid modifier specification: '" + spec.substring(next, end) + "'; expecting '=' for value assignment");
                 String key = spec.substring(next + 1, mid).trim();
                 String value = spec.substring(mid + 1, end).trim();
                 switch (key)
                 {
                     default: throw new IllegalArgumentException("Invalid modifier specification: unrecognised property '" + key + '\'');
                     case "retries":
                         retries = Integer.parseInt(value);
                         if (retries < 0)
                             throw new IllegalArgumentException("retries must be non-negative (retries=" + value + " supplied)");
                         break;
                     case "attempts":
                         retries = Integer.parseInt(value);
                         if (retries < 0)
                             throw new IllegalArgumentException("Must permit at least one attempt (attempts=" + value + " supplied)");
                         break;
                     case "rnd":
                         if (randomizer != null)
                             throw new IllegalArgumentException("Randomizer already specified, cannot re-specify: " + value);
                         randomizer = parseWaitRandomizer(value);
                         break;
                 }
                 end = next;
             }
             if (end != spec.length())
                 spec = spec.substring(0, end);
         }

         Matcher m = PARSE.matcher(spec);
         if (!m.matches())
             throw new IllegalArgumentException("Invalid specification: '" + spec + "'; does not match " + PARSE);

         long minMin = parseInMicros(m.group("minmin"), 0);
         long maxMax = parseInMicros(m.group("maxmax"), Long.MAX_VALUE);
         Wait max = TimeoutStrategy.parseWait(m.group("max"), latencies);
         String minSpec = m.group("min");
         Wait min = minSpec == null ? null : TimeoutStrategy.parseWait(minSpec, latencies);
         if (min == null && randomizer != null)
             throw new IllegalArgumentException("Invalid to specify randomiser when no range specified: '" + original + '\'');
         if (min instanceof Wait.Constant && minMin != 0)
             throw new IllegalArgumentException("Invalid to specify an absolute minimum constant when the min bound is itself a constant: '" + original + '\'');
         long maxMin = parseInMicros(m.group("maxmin"), Long.MAX_VALUE);
         if (min instanceof Wait.Constant && maxMin != Long.MAX_VALUE)
             throw new IllegalArgumentException("Invalid to specify an absolute max(min) constant when the min bound is itself a constant: '" + original + '\'');
         if (max instanceof Wait.Constant && maxMax != Long.MAX_VALUE)
             throw new IllegalArgumentException("Invalid to specify an absolute maximum constant when the max bound is itself a constant: '" + original + '\'');
         if (randomizer == null)
             randomizer = randomizers.uniform();
         return new RetryStrategy(randomizer, minMin, min, maxMin, max, maxMax, retries);
     }

     @VisibleForTesting
     protected static WaitRandomizer parseWaitRandomizer(String input)
     {
         Matcher m = RANDOMIZER.matcher(input);
         if (!m.matches())
             throw new IllegalArgumentException(input + " does not match" + RANDOMIZER);

         String exp;
         exp = m.group("exp");
         if (exp != null)
             return randomizers.exponential(Double.parseDouble(exp));
         exp = m.group("qexp");
         if (exp != null)
             return randomizers.quantizedExponential(Double.parseDouble(exp));
         return randomizers.uniform();
     }
 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	package org.apache.cassandra.service;

	import java.util.concurrent.ThreadLocalRandom;
	import java.util.concurrent.TimeUnit;
	import java.util.function.DoubleSupplier;
	import java.util.function.LongBinaryOperator;
	import java.util.regex.Matcher;
	import java.util.regex.Pattern;

	import javax.annotation.Nonnull;
	import javax.annotation.Nullable;

	import com.google.common.annotations.VisibleForTesting;

	import accord.utils.Invariants;
	import accord.utils.RandomSource;

	import org.apache.cassandra.service.TimeoutStrategy.LatencySourceFactory;
	import org.apache.cassandra.service.TimeoutStrategy.Wait;

	import static java.util.concurrent.TimeUnit.MICROSECONDS;
	import static java.util.concurrent.TimeUnit.NANOSECONDS;
	import static org.apache.cassandra.service.TimeoutStrategy.parseInMicros;
	import static org.apache.cassandra.utils.Clock.Global.nanoTime;

	/**
	* <p>A strategy for making retry timing decisions for operations.
	* The strategy is defined by six factors: <ul>
	* <li> {@link #minMinMicros}
	* <li> {@link #maxMaxMicros}
	* <li> {@link #min}
	* <li> {@link #max}
	* <li> {@link #waitRandomizer}
	* <li> {@link #maxAttempts}
	* </ul>
	*
	* <p>The first two represent the absolute upper and lower bound times we are permitted to produce as constants</p>
	* <p>The next two represent time periods, and may be defined dynamically based on a simple calculation over: <ul>
	* <li> {@code pX()} recent experienced latency distribution for successful operations,
	* e.g. {@code p50(rw)} the maximum of read and write median latencies,
	* {@code p999(r)} the 99.9th percentile of read latencies
	* <li> {@code attempts} the number of failed attempts made by the operation so far
	* <li> {@code constant} a user provided floating point constant
	* </ul>
	*
	* <p>Their calculation may take any of these forms
	* <li> constant {@code $constant$[mu]s}
	* <li> dynamic constant {@code pX() * constant}
	* <li> dynamic linear {@code pX() * constant * attempts}
	* <li> dynamic exponential {@code pX() * constant ^ attempts}
	* <li> Note: for dynamic exponential, attempts is subtracted by 1, such that times begin at {@code pX() * constant}.
	* e.g.
	* <li> {@code 10ms <= p50(rw)*0.66...p99(rw)}
	* <li> {@code 10ms <= p95(rw)*1.8^attempts <= 100ms}
	* <li> {@code 5ms <= p50(rw)*0.5}
	*
	* <p>These calculations are put together to construct a range from which we draw a random number.
	* The period we wait for {@code X} will be drawn so that {@code minMin <= min <= max <= maxMax}.
	*
	* <p>With the constraint that {@code max} must be {@code spread} greater than {@code min},
	* but no greater than its expression-defined maximum. {@code max} will be increased up until
	* this point, after which {@code min} will be decreased until this gap is imposed.
	*
	* <p>The {@link #waitRandomizer} property specifies the manner in which a random value is drawn from the range.
	* It is defined using one of the following specifiers:
	* <li> uniform
	* <li> exp($power$) or exponential($power$)
	* <li> qexp($power$) or qexponential($power$) or quantizedexponential($power$)
	*
	* The uniform specifier is self-explanatory, selecting all values in the range with equal probability.
	* The exponential specifier draws values towards the end of the range with higher probability, raising
	* a floating point number in the range [0..1.0) to the power provided, and translating the resulting value
	* to a uniform value in the range.
	* The quantized exponential specifier partitions the range into {@code attempts} buckets, then applies the pure
	* exponential approach to draw values from [0..attempts), before drawing a uniform value from the corresponding bucket
	*/
	public class RetryStrategy implements WaitStrategy
	{
	private static final Pattern RANDOMIZER = Pattern.compile(
	"uniform\|exp(onential)?[(](?<exp>[0-9.]+)[)]\|q(uantized)?exp(onential)?[(](?<qexp>[0-9.]+)[)]");

	static final Pattern PARSE = Pattern.compile(
	"(\\s(?<minmin>0\|[0-9]+[mu]?s)\\s<=)?" +
	"(\\s(?<min>[^=]+)([(]?\\s<=\\s(?<maxmin>0\|[0-9]+[mu]?s)\\s[)]?)?\\s*[.]{3})?" +
	"(\\s*(?<max>[^=]+))" +
	"(\\s<=\\s(?<maxmax>0\|[0-9]+[mu]?s))?");

	public static final WaitRandomizerFactory randomizers = new WaitRandomizerFactory(){};

	public interface WaitRandomizer
	{
	long wait(long min, long max, int attempts);
	}

	public static WaitRandomizerFactory randomizers(RandomSource random)
	{
	return new WaitRandomizerFactory()
	{
	@Override public LongBinaryOperator uniformLongSupplier() { return random::nextLong; }
	@Override public DoubleSupplier uniformDoubleSupplier() { return random::nextDouble; }
	};
	}

	public interface WaitRandomizerFactory
	{
	default LongBinaryOperator uniformLongSupplier() { return (min, max) -> ThreadLocalRandom.current().nextLong(min, max); } // DO NOT USE METHOD HANDLES (want to fetch afresh each time)
	default DoubleSupplier uniformDoubleSupplier() { return () -> ThreadLocalRandom.current().nextDouble(); }

	default WaitRandomizer uniform() { return new Uniform(uniformLongSupplier()); }
	default WaitRandomizer exponential(double power) { return new Exponential(uniformLongSupplier(), uniformDoubleSupplier(), power); }
	default WaitRandomizer quantizedExponential(double power) { return new QuantizedExponential(uniformLongSupplier(), uniformDoubleSupplier(), power); }

	class Uniform implements WaitRandomizer
	{
	final LongBinaryOperator uniformLong;

	public Uniform(LongBinaryOperator uniformLong)
	{
	this.uniformLong = uniformLong;
	}

	@Override
	public long wait(long min, long max, int attempts)
	{
	return uniformLong.applyAsLong(min, max);
	}
	}

	abstract class AbstractExponential implements WaitRandomizer
	{
	final LongBinaryOperator uniformLong;
	final DoubleSupplier uniformDouble;
	final double power;

	public AbstractExponential(LongBinaryOperator uniformLong, DoubleSupplier uniformDouble, double power)
	{
	this.uniformLong = uniformLong;
	this.uniformDouble = uniformDouble;
	this.power = power;
	}
	}

	class Exponential extends AbstractExponential
	{
	public Exponential(LongBinaryOperator uniformLong, DoubleSupplier uniformDouble, double power)
	{
	super(uniformLong, uniformDouble, power);
	}

	@Override
	public long wait(long min, long max, int attempts)
	{
	if (attempts == 1)
	return uniformLong.applyAsLong(min, max);

	double p = uniformDouble.getAsDouble();
	long delta = max - min;
	delta *= Math.pow(p, power);
	return max - delta;
	}
	}

	class QuantizedExponential extends AbstractExponential
	{
	public QuantizedExponential(LongBinaryOperator uniformLong, DoubleSupplier uniformDouble, double power)
	{
	super(uniformLong, uniformDouble, power);
	}

	@Override
	public long wait(long min, long max, int attempts)
	{
	long quanta = (max - min) / attempts;
	if (attempts == 1 \|\| quanta == 0)
	return uniformLong.applyAsLong(min, max);

	double p = uniformDouble.getAsDouble();
	int base = (int) (attempts * Math.pow(p, power));
	return max - ThreadLocalRandom.current().nextLong(quanta * base, quanta * (base + 1));
	}
	}
	}

	public final WaitRandomizer waitRandomizer;
	public final long minMinMicros, maxMinMicros, maxMaxMicros;
	public final @Nullable Wait min;
	public final @Nonnull Wait max;
	public final int maxAttempts;

	protected RetryStrategy(WaitRandomizer waitRandomizer, long minMinMicros, Wait min, long maxMinMicros, Wait max, long maxMaxMicros, int retries)
	{
	this.waitRandomizer = waitRandomizer;
	this.minMinMicros = minMinMicros;
	this.min = min;
	this.maxMinMicros = maxMinMicros;
	this.max = max;
	this.maxMaxMicros = maxMaxMicros;
	this.maxAttempts = retries == Integer.MAX_VALUE ? Integer.MAX_VALUE : retries + 1;
	Invariants.require(maxAttempts >= 1);
	}

	public long computeWaitUntil(int attempts)
	{
	long wait = computeWait(attempts, NANOSECONDS);
	if (wait < 0)
	return -1;
	return nanoTime() + wait;
	}

	public long computeWait(int attempt, TimeUnit units)
	{
	if (attempt > maxAttempts)
	return -1;

	long result;
	if (min == null)
	{
	result = max.getMicros(attempt);
	}
	else
	{
	long min = this.min.getMicros(attempt);
	if (min > maxMinMicros)
	min = maxMinMicros;
	long max = this.max.getMicros(attempt);
	if (max > maxMaxMicros)
	max = maxMaxMicros;
	result = min >= max ? min : waitRandomizer.wait(min, max, attempt);
	}

	if (result > maxMaxMicros) result = maxMaxMicros;
	if (result < minMinMicros) result = minMinMicros;
	return units.convert(result, MICROSECONDS);
	}

	public long computeMaxWait(int attempt, TimeUnit units)
	{
	return units.convert(max.getMaxMicros(attempt), MICROSECONDS);
	}

	public static RetryStrategy parse(String spec, LatencySourceFactory latencies)
	{
	return parse(spec, latencies, null);
	}

	public static RetryStrategy parse(String spec, LatencySourceFactory latencies, WaitRandomizer randomizer)
	{
	String original = spec;
	int retries = Integer.MAX_VALUE;
	int end = spec.length();
	{
	int next;
	while ((next = spec.lastIndexOf(',', end - 1)) >= 0)
	{
	int mid = spec.indexOf('=', next + 1);
	if (mid <= next \|\| mid >= end)
	throw new IllegalArgumentException("Invalid modifier specification: '" + spec.substring(next, end) + "'; expecting '=' for value assignment");
	String key = spec.substring(next + 1, mid).trim();
	String value = spec.substring(mid + 1, end).trim();
	switch (key)
	{
	default: throw new IllegalArgumentException("Invalid modifier specification: unrecognised property '" + key + '\'');
	case "retries":
	retries = Integer.parseInt(value);
	if (retries < 0)
	throw new IllegalArgumentException("retries must be non-negative (retries=" + value + " supplied)");
	break;
	case "attempts":
	retries = Integer.parseInt(value);
	if (retries < 0)
	throw new IllegalArgumentException("Must permit at least one attempt (attempts=" + value + " supplied)");
	break;
	case "rnd":
	if (randomizer != null)
	throw new IllegalArgumentException("Randomizer already specified, cannot re-specify: " + value);
	randomizer = parseWaitRandomizer(value);
	break;
	}
	end = next;
	}
	if (end != spec.length())
	spec = spec.substring(0, end);
	}

	Matcher m = PARSE.matcher(spec);
	if (!m.matches())
	throw new IllegalArgumentException("Invalid specification: '" + spec + "'; does not match " + PARSE);

	long minMin = parseInMicros(m.group("minmin"), 0);
	long maxMax = parseInMicros(m.group("maxmax"), Long.MAX_VALUE);
	Wait max = TimeoutStrategy.parseWait(m.group("max"), latencies);
	String minSpec = m.group("min");
	Wait min = minSpec == null ? null : TimeoutStrategy.parseWait(minSpec, latencies);
	if (min == null && randomizer != null)
	throw new IllegalArgumentException("Invalid to specify randomiser when no range specified: '" + original + '\'');
	if (min instanceof Wait.Constant && minMin != 0)
	throw new IllegalArgumentException("Invalid to specify an absolute minimum constant when the min bound is itself a constant: '" + original + '\'');
	long maxMin = parseInMicros(m.group("maxmin"), Long.MAX_VALUE);
	if (min instanceof Wait.Constant && maxMin != Long.MAX_VALUE)
	throw new IllegalArgumentException("Invalid to specify an absolute max(min) constant when the min bound is itself a constant: '" + original + '\'');
	if (max instanceof Wait.Constant && maxMax != Long.MAX_VALUE)
	throw new IllegalArgumentException("Invalid to specify an absolute maximum constant when the max bound is itself a constant: '" + original + '\'');
	if (randomizer == null)
	randomizer = randomizers.uniform();
	return new RetryStrategy(randomizer, minMin, min, maxMin, max, maxMax, retries);
	}

	@VisibleForTesting
	protected static WaitRandomizer parseWaitRandomizer(String input)
	{
	Matcher m = RANDOMIZER.matcher(input);
	if (!m.matches())
	throw new IllegalArgumentException(input + " does not match" + RANDOMIZER);

	String exp;
	exp = m.group("exp");
	if (exp != null)
	return randomizers.exponential(Double.parseDouble(exp));
	exp = m.group("qexp");
	if (exp != null)
	return randomizers.quantizedExponential(Double.parseDouble(exp));
	return randomizers.uniform();
	}
	}