get CDF functions for quantiles sketch
diff --git a/src/main/java/com/yahoo/sketches/hive/quantiles/GetCdfFromDoublesSketchUDF.java b/src/main/java/com/yahoo/sketches/hive/quantiles/GetCdfFromDoublesSketchUDF.java
new file mode 100644
index 0000000..f6ba456
--- /dev/null
+++ b/src/main/java/com/yahoo/sketches/hive/quantiles/GetCdfFromDoublesSketchUDF.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2019, Verizon Media.
+ * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
+ */
+
+package com.yahoo.sketches.hive.quantiles;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.io.BytesWritable;
+
+import com.yahoo.memory.Memory;
+import com.yahoo.sketches.quantiles.DoublesSketch;
+
+@Description(
+ name = "GetCDF",
+ value = "_FUNC_(sketch, split points...)",
+ extended = "Returns an approximation to the Cumulative Distribution Function (CDF)"
+ + " from a sketch given a set of split points (values)."
+ + " Split points are an array of M unique, monotonically increasing values"
+ + " that divide the real number line into M+1 consecutive disjoint intervals."
+ + " The function returns an array of M+1 double valuess, the first M of which are approximations"
+ + " to the ranks of the corresponding split points (fraction of input stream values that are less"
+ + " than a split point). The last value is always 1."
+ + " CDF can also be viewed as a cumulative version of PMF.")
+public class GetCdfFromDoublesSketchUDF extends UDF {
+
+ /**
+ * Returns a list of ranks (CDF) from a given sketch
+ * @param serializedSketch serialized sketch
+ * @param splitPoints list of unique and monotonically increasing values
+ * @return list of fractions from 0 to 1
+ */
+ public List<Double> evaluate(final BytesWritable serializedSketch, final Double... splitPoints) {
+ if (serializedSketch == null) { return null; }
+ final DoublesSketch sketch = DoublesSketch.wrap(Memory.wrap(serializedSketch.getBytes()));
+ final double[] cdf = sketch.getCDF(Util.objectsToPrimitives(splitPoints));
+ if (cdf == null) { return null; }
+ return Util.primitivesToList(cdf);
+ }
+
+}
diff --git a/src/main/java/com/yahoo/sketches/hive/quantiles/GetCdfFromStringsSketchUDF.java b/src/main/java/com/yahoo/sketches/hive/quantiles/GetCdfFromStringsSketchUDF.java
new file mode 100644
index 0000000..55315da
--- /dev/null
+++ b/src/main/java/com/yahoo/sketches/hive/quantiles/GetCdfFromStringsSketchUDF.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2019, Verizon Media.
+ * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
+ */
+
+package com.yahoo.sketches.hive.quantiles;
+
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.io.BytesWritable;
+
+import com.yahoo.memory.Memory;
+import com.yahoo.sketches.ArrayOfStringsSerDe;
+import com.yahoo.sketches.quantiles.ItemsSketch;
+
+@Description(
+ name = "GetCDF",
+ value = "_FUNC_(sketch, split points...)",
+ extended = "Returns an approximation to the Cumulative Distribution Function (CDF)"
+ + " from a sketch given a set of split points (values)."
+ + " Split points are an array of M unique, monotonically increasing values"
+ + " that divide the domain into M+1 consecutive disjoint intervals."
+ + " The function returns an array of M+1 double valuess, the first M of which are approximations"
+ + " to the ranks of the corresponding split points (fraction of input stream values that are less"
+ + " than a split point). The last value is always 1."
+ + " CDF can also be viewed as a cumulative version of PMF.")
+public class GetCdfFromStringsSketchUDF extends UDF {
+
+ /**
+ * Returns a list of ranks (CDF) from a given sketch
+ * @param serializedSketch serialized sketch
+ * @param splitPoints list of unique and monotonically increasing values
+ * @return list of fractions from 0 to 1
+ */
+ public List<Double> evaluate(final BytesWritable serializedSketch, final String... splitPoints) {
+ if (serializedSketch == null) { return null; }
+ final ItemsSketch<String> sketch = ItemsSketch.getInstance(
+ Memory.wrap(serializedSketch.getBytes()),
+ Comparator.naturalOrder(),
+ new ArrayOfStringsSerDe()
+ );
+ final double[] cdf = sketch.getCDF(splitPoints);
+ if (cdf == null) { return null; }
+ return Util.primitivesToList(cdf);
+ }
+
+}
diff --git a/src/test/java/com/yahoo/sketches/hive/quantiles/GetCdfFromDoublesSketchUDFTest.java b/src/test/java/com/yahoo/sketches/hive/quantiles/GetCdfFromDoublesSketchUDFTest.java
new file mode 100644
index 0000000..3506506
--- /dev/null
+++ b/src/test/java/com/yahoo/sketches/hive/quantiles/GetCdfFromDoublesSketchUDFTest.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2019, Verizon Media.
+ * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
+ */
+
+package com.yahoo.sketches.hive.quantiles;
+
+import java.util.List;
+
+import org.apache.hadoop.io.BytesWritable;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import com.yahoo.sketches.quantiles.DoublesSketch;
+import com.yahoo.sketches.quantiles.UpdateDoublesSketch;
+
+public class GetCdfFromDoublesSketchUDFTest {
+
+ @Test
+ public void nullSketch() {
+ List<Double> result = new GetCdfFromDoublesSketchUDF().evaluate(null, 0.0);
+ Assert.assertNull(result);
+ }
+
+ @Test
+ public void emptyListOfSplitPoints() {
+ UpdateDoublesSketch sketch = DoublesSketch.builder().build();
+ sketch.update(1);
+ sketch.update(2);
+ sketch.update(3);
+ List<Double> result = new GetCdfFromDoublesSketchUDF().evaluate(new BytesWritable(sketch.toByteArray()));
+ Assert.assertNotNull(result);
+ Assert.assertEquals(result.size(), 1);
+ Assert.assertEquals(result.get(0), 1.0);
+ }
+
+ @Test
+ public void emptySketch() {
+ UpdateDoublesSketch sketch = DoublesSketch.builder().build();
+ List<Double> result = new GetCdfFromDoublesSketchUDF().evaluate(new BytesWritable(sketch.toByteArray()), 0.0);
+ Assert.assertNull(result);
+ }
+
+ @Test
+ public void normalCase() {
+ UpdateDoublesSketch sketch = DoublesSketch.builder().build();
+ sketch.update(1);
+ sketch.update(2);
+ sketch.update(3);
+ sketch.update(4);
+ List<Double> result = new GetCdfFromDoublesSketchUDF().evaluate(new BytesWritable(sketch.toByteArray()), 1.0, 3.0, 4.0);
+ Assert.assertNotNull(result);
+ Assert.assertEquals(result.size(), 4);
+ Assert.assertEquals(result.get(0), 0.0);
+ Assert.assertEquals(result.get(1), 0.5);
+ Assert.assertEquals(result.get(2), 0.75);
+ Assert.assertEquals(result.get(3), 1.0);
+ }
+
+}
diff --git a/src/test/java/com/yahoo/sketches/hive/quantiles/GetCdfFromStringsSketchUDFTest.java b/src/test/java/com/yahoo/sketches/hive/quantiles/GetCdfFromStringsSketchUDFTest.java
new file mode 100644
index 0000000..e03847d
--- /dev/null
+++ b/src/test/java/com/yahoo/sketches/hive/quantiles/GetCdfFromStringsSketchUDFTest.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2019, Verizon Media.
+ * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
+ */
+
+package com.yahoo.sketches.hive.quantiles;
+
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.hadoop.io.BytesWritable;
+
+import org.testng.annotations.Test;
+import org.testng.Assert;
+
+import com.yahoo.sketches.ArrayOfItemsSerDe;
+import com.yahoo.sketches.ArrayOfStringsSerDe;
+import com.yahoo.sketches.quantiles.ItemsSketch;
+
+public class GetCdfFromStringsSketchUDFTest {
+
+ static final Comparator<String> comparator = Comparator.naturalOrder();
+ static final ArrayOfItemsSerDe<String> serDe = new ArrayOfStringsSerDe();
+
+ @Test
+ public void nullSketch() {
+ List<Double> result = new GetCdfFromStringsSketchUDF().evaluate(null, "");
+ Assert.assertNull(result);
+ }
+
+ @Test
+ public void emptyListOfSplitPoints() {
+ ItemsSketch<String> sketch = ItemsSketch.getInstance(comparator);
+ sketch.update("a");
+ sketch.update("b");
+ sketch.update("c");
+ List<Double> result = new GetCdfFromStringsSketchUDF().evaluate(new BytesWritable(sketch.toByteArray(serDe)));
+ Assert.assertNotNull(result);
+ Assert.assertEquals(result.size(), 1);
+ Assert.assertEquals(result.get(0), 1.0);
+ }
+
+ @Test
+ public void emptySketch() {
+ ItemsSketch<String> sketch = ItemsSketch.getInstance(comparator);
+ List<Double> result = new GetCdfFromStringsSketchUDF().evaluate(new BytesWritable(sketch.toByteArray(serDe)), "a");
+ Assert.assertNull(result);
+ }
+
+ @Test
+ public void normalCase() {
+ ItemsSketch<String> sketch = ItemsSketch.getInstance(comparator);
+ sketch.update("a");
+ sketch.update("b");
+ sketch.update("c");
+ sketch.update("d");
+ List<Double> result = new GetCdfFromStringsSketchUDF().evaluate(new BytesWritable(sketch.toByteArray(serDe)), "a", "c", "d");
+ Assert.assertNotNull(result);
+ Assert.assertEquals(result.size(), 4);
+ Assert.assertEquals(result.get(0), 0.0);
+ Assert.assertEquals(result.get(1), 0.5);
+ Assert.assertEquals(result.get(2), 0.75);
+ Assert.assertEquals(result.get(3), 1.0);
+ }
+
+}
