Merge pull request #170 from apache/tuple_sketch
Tuple sketch
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c98c4a2..ff1d4a5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -95,6 +95,7 @@
add_subdirectory(fi)
add_subdirectory(theta)
add_subdirectory(sampling)
+add_subdirectory(tuple)
if (WITH_PYTHON)
add_subdirectory(python)
diff --git a/LICENSE b/LICENSE
index a7ccd16..5947c4b 100644
--- a/LICENSE
+++ b/LICENSE
@@ -306,5 +306,6 @@
* Placed in the Public Domain by Sean Eron Anderson
Code Locations
- * common/include/CommonUtil.h
- that is adapted from the above.
\ No newline at end of file
+ * common/include/ceiling_power_of_2.hpp
+ that is adapted from the above.
+
\ No newline at end of file
diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt
index 738e4b3..7383acc 100644
--- a/common/CMakeLists.txt
+++ b/common/CMakeLists.txt
@@ -37,5 +37,9 @@
${CMAKE_CURRENT_SOURCE_DIR}/include/serde.hpp
${CMAKE_CURRENT_SOURCE_DIR}/include/count_zeros.hpp
${CMAKE_CURRENT_SOURCE_DIR}/include/inv_pow2_table.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/binomial_bounds.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/conditional_back_inserter.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/conditional_forward.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/ceiling_power_of_2.hpp
)
diff --git a/theta/include/binomial_bounds.hpp b/common/include/binomial_bounds.hpp
similarity index 100%
rename from theta/include/binomial_bounds.hpp
rename to common/include/binomial_bounds.hpp
diff --git a/common/include/ceiling_power_of_2.hpp b/common/include/ceiling_power_of_2.hpp
new file mode 100644
index 0000000..b149436
--- /dev/null
+++ b/common/include/ceiling_power_of_2.hpp
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef CEILING_POWER_OF_2_HPP_
+#define CEILING_POWER_OF_2_HPP_
+
+#include <cstdint>
+
+namespace datasketches {
+
+// compute the next highest power of 2 of 32-bit n
+// taken from https://graphics.stanford.edu/~seander/bithacks.html
+static inline uint32_t ceiling_power_of_2(uint32_t n) {
+ --n;
+ n |= n >> 1;
+ n |= n >> 2;
+ n |= n >> 4;
+ n |= n >> 8;
+ n |= n >> 16;
+ return ++n;
+}
+
+} /* namespace datasketches */
+
+#endif // CEILING_POWER_OF_2_HPP_
diff --git a/common/include/common_defs.hpp b/common/include/common_defs.hpp
index 7a7b40d..ffb3f19 100644
--- a/common/include/common_defs.hpp
+++ b/common/include/common_defs.hpp
@@ -31,6 +31,21 @@
template<typename A> using AllocChar = typename std::allocator_traits<A>::template rebind_alloc<char>;
template<typename A> using string = std::basic_string<char, std::char_traits<char>, AllocChar<A>>;
+// utility function to hide unused compiler warning
+// usually has no additional cost
+template<typename T> void unused(T&&...) {}
+
+// common helping functions
+// TODO: find a better place for them
+
+constexpr uint8_t log2(uint32_t n) {
+ return (n > 1) ? 1 + log2(n >> 1) : 0;
+}
+
+constexpr uint8_t lg_size_from_count(uint32_t n, double load_factor) {
+ return log2(n) + ((n > static_cast<uint32_t>((1 << (log2(n) + 1)) * load_factor)) ? 2 : 1);
+}
+
} // namespace
#endif // _COMMON_DEFS_HPP_
diff --git a/theta/include/conditional_back_inserter.hpp b/common/include/conditional_back_inserter.hpp
similarity index 90%
rename from theta/include/conditional_back_inserter.hpp
rename to common/include/conditional_back_inserter.hpp
index 9b33833..8a2b38b 100644
--- a/theta/include/conditional_back_inserter.hpp
+++ b/common/include/conditional_back_inserter.hpp
@@ -40,11 +40,16 @@
return *this;
}
- conditional_back_insert_iterator& operator=(typename Container::const_reference value) {
+ conditional_back_insert_iterator& operator=(const typename Container::value_type& value) {
if (p(value)) std::back_insert_iterator<Container>::operator=(value);
return *this;
}
+ conditional_back_insert_iterator& operator=(typename Container::value_type&& value) {
+ if (p(value)) std::back_insert_iterator<Container>::operator=(std::move(value));
+ return *this;
+ }
+
conditional_back_insert_iterator& operator*() { return *this; }
conditional_back_insert_iterator& operator++() { return *this; }
conditional_back_insert_iterator& operator++(int) { return *this; }
diff --git a/common/include/conditional_forward.hpp b/common/include/conditional_forward.hpp
new file mode 100644
index 0000000..4648b85
--- /dev/null
+++ b/common/include/conditional_forward.hpp
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef CONDITIONAL_FORWARD_HPP_
+#define CONDITIONAL_FORWARD_HPP_
+
+#include <type_traits>
+
+namespace datasketches {
+
+// Forward type T2 as rvalue reference if type T1 is rvalue reference
+
+template<typename T1, typename T2>
+using fwd_type = typename std::conditional<std::is_lvalue_reference<T1>::value,
+ T2, typename std::remove_reference<T2>::type&&>::type;
+
+template<typename T1, typename T2>
+fwd_type<T1, T2> conditional_forward(T2&& value) {
+ return std::forward<fwd_type<T1, T2>>(std::forward<T2>(value));
+}
+
+// Forward container as iterators
+
+template<typename Container>
+auto forward_begin(Container&& c) ->
+typename std::enable_if<std::is_lvalue_reference<Container>::value, decltype(c.begin())>::type
+{
+ return c.begin();
+}
+
+template<typename Container>
+auto forward_begin(Container&& c) ->
+typename std::enable_if<!std::is_lvalue_reference<Container>::value, decltype(std::make_move_iterator(c.begin()))>::type
+{
+ return std::make_move_iterator(c.begin());
+}
+
+template<typename Container>
+auto forward_end(Container&& c) ->
+typename std::enable_if<std::is_lvalue_reference<Container>::value, decltype(c.end())>::type
+{
+ return c.end();
+}
+
+template<typename Container>
+auto forward_end(Container&& c) ->
+typename std::enable_if<!std::is_lvalue_reference<Container>::value, decltype(std::make_move_iterator(c.end()))>::type
+{
+ return std::make_move_iterator(c.end());
+}
+
+} /* namespace datasketches */
+
+#endif
diff --git a/common/include/serde.hpp b/common/include/serde.hpp
index d0819d8..73e0901 100644
--- a/common/include/serde.hpp
+++ b/common/include/serde.hpp
@@ -33,13 +33,13 @@
// serialize and deserialize
template<typename T, typename Enable = void> struct serde {
// stream serialization
- void serialize(std::ostream& os, const T* items, unsigned num);
- void deserialize(std::istream& is, T* items, unsigned num); // items allocated but not initialized
+ void serialize(std::ostream& os, const T* items, unsigned num) const;
+ void deserialize(std::istream& is, T* items, unsigned num) const; // items allocated but not initialized
// raw bytes serialization
- size_t size_of_item(const T& item);
- size_t serialize(void* ptr, size_t capacity, const T* items, unsigned num);
- size_t deserialize(const void* ptr, size_t capacity, T* items, unsigned num); // items allocated but not initialized
+ size_t size_of_item(const T& item) const;
+ size_t serialize(void* ptr, size_t capacity, const T* items, unsigned num) const;
+ size_t deserialize(const void* ptr, size_t capacity, T* items, unsigned num) const; // items allocated but not initialized
};
// serde for all fixed-size arithmetic types (int and float of different sizes)
@@ -47,7 +47,7 @@
// with LongsSketch and ItemsSketch<Long> with ArrayOfLongsSerDe in Java
template<typename T>
struct serde<T, typename std::enable_if<std::is_arithmetic<T>::value>::type> {
- void serialize(std::ostream& os, const T* items, unsigned num) {
+ void serialize(std::ostream& os, const T* items, unsigned num) const {
bool failure = false;
try {
os.write(reinterpret_cast<const char*>(items), sizeof(T) * num);
@@ -58,7 +58,7 @@
throw std::runtime_error("error writing to std::ostream with " + std::to_string(num) + " items");
}
}
- void deserialize(std::istream& is, T* items, unsigned num) {
+ void deserialize(std::istream& is, T* items, unsigned num) const {
bool failure = false;
try {
is.read((char*)items, sizeof(T) * num);
@@ -70,16 +70,16 @@
}
}
- size_t size_of_item(const T&) {
+ size_t size_of_item(const T&) const {
return sizeof(T);
}
- size_t serialize(void* ptr, size_t capacity, const T* items, unsigned num) {
+ size_t serialize(void* ptr, size_t capacity, const T* items, unsigned num) const {
const size_t bytes_written = sizeof(T) * num;
check_memory_size(bytes_written, capacity);
memcpy(ptr, items, bytes_written);
return bytes_written;
}
- size_t deserialize(const void* ptr, size_t capacity, T* items, unsigned num) {
+ size_t deserialize(const void* ptr, size_t capacity, T* items, unsigned num) const {
const size_t bytes_read = sizeof(T) * num;
check_memory_size(bytes_read, capacity);
memcpy(items, ptr, bytes_read);
@@ -94,7 +94,7 @@
// which may be too wasteful. Treat this as an example.
template<>
struct serde<std::string> {
- void serialize(std::ostream& os, const std::string* items, unsigned num) {
+ void serialize(std::ostream& os, const std::string* items, unsigned num) const {
unsigned i = 0;
bool failure = false;
try {
@@ -110,7 +110,7 @@
throw std::runtime_error("error writing to std::ostream at item " + std::to_string(i));
}
}
- void deserialize(std::istream& is, std::string* items, unsigned num) {
+ void deserialize(std::istream& is, std::string* items, unsigned num) const {
unsigned i = 0;
bool failure = false;
try {
@@ -137,10 +137,10 @@
throw std::runtime_error("error reading from std::istream at item " + std::to_string(i));
}
}
- size_t size_of_item(const std::string& item) {
+ size_t size_of_item(const std::string& item) const {
return sizeof(uint32_t) + item.size();
}
- size_t serialize(void* ptr, size_t capacity, const std::string* items, unsigned num) {
+ size_t serialize(void* ptr, size_t capacity, const std::string* items, unsigned num) const {
size_t bytes_written = 0;
for (unsigned i = 0; i < num; ++i) {
const uint32_t length = items[i].size();
@@ -154,7 +154,7 @@
}
return bytes_written;
}
- size_t deserialize(const void* ptr, size_t capacity, std::string* items, unsigned num) {
+ size_t deserialize(const void* ptr, size_t capacity, std::string* items, unsigned num) const {
size_t bytes_read = 0;
unsigned i = 0;
bool failure = false;
diff --git a/common/test/test_allocator.cpp b/common/test/test_allocator.cpp
index 02c5654..47389ff 100644
--- a/common/test/test_allocator.cpp
+++ b/common/test/test_allocator.cpp
@@ -24,4 +24,8 @@
// global variable to keep track of allocated size
long long test_allocator_total_bytes = 0;
+// global variable to keep track of net allocations
+// (number of allocations minus number of deallocations)
+long long test_allocator_net_allocations = 0;
+
} /* namespace datasketches */
diff --git a/common/test/test_allocator.hpp b/common/test/test_allocator.hpp
index 41dacfd..b383af6 100644
--- a/common/test/test_allocator.hpp
+++ b/common/test/test_allocator.hpp
@@ -28,6 +28,7 @@
namespace datasketches {
extern long long test_allocator_total_bytes;
+extern long long test_allocator_net_allocations;
template <class T> class test_allocator {
public:
@@ -46,7 +47,10 @@
test_allocator(const test_allocator&) {}
template <class U>
test_allocator(const test_allocator<U>&) {}
+ test_allocator(test_allocator&&) {}
~test_allocator() {}
+ test_allocator& operator=(const test_allocator&) { return *this; }
+ test_allocator& operator=(test_allocator&&) { return *this; }
pointer address(reference x) const { return &x; }
const_pointer address(const_reference x) const {
@@ -57,12 +61,14 @@
void* p = new char[n * sizeof(value_type)];
if (!p) throw std::bad_alloc();
test_allocator_total_bytes += n * sizeof(value_type);
+ ++test_allocator_net_allocations;
return static_cast<pointer>(p);
}
void deallocate(pointer p, size_type n) {
if (p) delete[] (char*) p;
test_allocator_total_bytes -= n * sizeof(value_type);
+ --test_allocator_net_allocations;
}
size_type max_size() const {
@@ -74,9 +80,6 @@
new(p) value_type(std::forward<Args>(args)...);
}
void destroy(pointer p) { p->~value_type(); }
-
-private:
- void operator=(const test_allocator&);
};
template<> class test_allocator<void> {
diff --git a/common/test/test_type.hpp b/common/test/test_type.hpp
index abbd8f3..18be598 100644
--- a/common/test/test_type.hpp
+++ b/common/test/test_type.hpp
@@ -24,38 +24,51 @@
namespace datasketches {
-class test_type {
+template<typename A>
+class test_type_alloc {
static const bool DEBUG = false;
public:
// no default constructor should be required
- test_type(int value): value(value) {
+ test_type_alloc(int value): value_ptr(A().allocate(1)) {
if (DEBUG) std::cerr << "test_type constructor" << std::endl;
+ *value_ptr = value;
}
- ~test_type() {
+ ~test_type_alloc() {
if (DEBUG) std::cerr << "test_type destructor" << std::endl;
+ if (value_ptr != nullptr) A().deallocate(value_ptr, 1);
}
- test_type(const test_type& other): value(other.value) {
+ test_type_alloc(const test_type_alloc& other): value_ptr(A().allocate(1)) {
if (DEBUG) std::cerr << "test_type copy constructor" << std::endl;
+ *value_ptr = *other.value_ptr;
}
// noexcept is important here so that, for instance, std::vector could move this type
- test_type(test_type&& other) noexcept : value(other.value) {
+ test_type_alloc(test_type_alloc&& other) noexcept : value_ptr(nullptr) {
if (DEBUG) std::cerr << "test_type move constructor" << std::endl;
+ if (DEBUG && other.value_ptr == nullptr) std::cerr << "moving null" << std::endl;
+ std::swap(value_ptr, other.value_ptr);
}
- test_type& operator=(const test_type& other) {
+ test_type_alloc& operator=(const test_type_alloc& other) {
if (DEBUG) std::cerr << "test_type copy assignment" << std::endl;
- value = other.value;
+ if (DEBUG && value_ptr == nullptr) std::cerr << "nullptr" << std::endl;
+ *value_ptr = *other.value_ptr;
return *this;
}
- test_type& operator=(test_type&& other) {
+ test_type_alloc& operator=(test_type_alloc&& other) {
if (DEBUG) std::cerr << "test_type move assignment" << std::endl;
- value = other.value;
+ if (DEBUG && other.value_ptr == nullptr) std::cerr << "moving null" << std::endl;
+ std::swap(value_ptr, other.value_ptr);
return *this;
}
- int get_value() const { return value; }
+ int get_value() const {
+ if (value_ptr == nullptr) std::cerr << "null" << std::endl;
+ return *value_ptr;
+ }
private:
- int value;
+ int* value_ptr;
};
+using test_type = test_type_alloc<std::allocator<int>>;
+
struct test_type_hash {
std::size_t operator()(const test_type& a) const {
return std::hash<int>()(a.get_value());
@@ -75,22 +88,43 @@
};
struct test_type_serde {
- void serialize(std::ostream& os, const test_type* items, unsigned num) {
+ void serialize(std::ostream& os, const test_type* items, unsigned num) const {
for (unsigned i = 0; i < num; i++) {
const int value = items[i].get_value();
os.write((char*)&value, sizeof(value));
}
}
- void deserialize(std::istream& is, test_type* items, unsigned num) {
+ void deserialize(std::istream& is, test_type* items, unsigned num) const {
for (unsigned i = 0; i < num; i++) {
int value;
is.read((char*)&value, sizeof(value));
new (&items[i]) test_type(value);
}
}
- size_t size_of_item(const test_type&) {
+ size_t size_of_item(const test_type&) const {
return sizeof(int);
}
+ size_t serialize(void* ptr, size_t capacity, const test_type* items, unsigned num) const {
+ const size_t bytes_written = sizeof(int) * num;
+ check_memory_size(bytes_written, capacity);
+ for (unsigned i = 0; i < num; ++i) {
+ const int value = items[i].get_value();
+ memcpy(ptr, &value, sizeof(int));
+ ptr = static_cast<char*>(ptr) + sizeof(int);
+ }
+ return bytes_written;
+ }
+ size_t deserialize(const void* ptr, size_t capacity, test_type* items, unsigned num) const {
+ const size_t bytes_read = sizeof(int) * num;
+ check_memory_size(bytes_read, capacity);
+ for (unsigned i = 0; i < num; ++i) {
+ int value;
+ memcpy(&value, ptr, sizeof(int));
+ new (&items[i]) test_type(value);
+ ptr = static_cast<const char*>(ptr) + sizeof(int);
+ }
+ return bytes_read;
+ }
};
std::ostream& operator<<(std::ostream& os, const test_type& a) {
diff --git a/hll/include/HllUtil.hpp b/hll/include/HllUtil.hpp
index 1bea84a..ec0ddf2 100644
--- a/hll/include/HllUtil.hpp
+++ b/hll/include/HllUtil.hpp
@@ -24,6 +24,7 @@
#include "RelativeErrorTables.hpp"
#include "count_zeros.hpp"
#include "common_defs.hpp"
+#include "ceiling_power_of_2.hpp"
#include <cmath>
#include <stdexcept>
@@ -212,19 +213,6 @@
return conv.doubleVal;
}
-// compute the next highest power of 2 of 32-bit n
-// taken from https://graphics.stanford.edu/~seander/bithacks.html
-template<typename A>
-inline uint32_t HllUtil<A>::ceilingPowerOf2(uint32_t n) {
- --n;
- n |= n >> 1;
- n |= n >> 2;
- n |= n >> 4;
- n |= n >> 8;
- n |= n >> 16;
- return ++n;
-}
-
template<typename A>
inline uint32_t HllUtil<A>::simpleIntLog2(uint32_t n) {
if (n == 0) {
@@ -237,7 +225,7 @@
inline int HllUtil<A>::computeLgArrInts(hll_mode mode, int count, int lgConfigK) {
// assume value missing and recompute
if (mode == LIST) { return HllUtil<A>::LG_INIT_LIST_SIZE; }
- int ceilPwr2 = HllUtil<A>::ceilingPowerOf2(count);
+ int ceilPwr2 = ceiling_power_of_2(count);
if ((HllUtil<A>::RESIZE_DENOM * count) > (HllUtil<A>::RESIZE_NUMER * ceilPwr2)) { ceilPwr2 <<= 1;}
if (mode == SET) {
return fmax(HllUtil<A>::LG_INIT_SET_SIZE, HllUtil<A>::simpleIntLog2(ceilPwr2));
@@ -248,4 +236,4 @@
}
-#endif /* _HLLUTIL_HPP_ */
\ No newline at end of file
+#endif /* _HLLUTIL_HPP_ */
diff --git a/sampling/include/var_opt_sketch.hpp b/sampling/include/var_opt_sketch.hpp
index 848a253..4572ea0 100644
--- a/sampling/include/var_opt_sketch.hpp
+++ b/sampling/include/var_opt_sketch.hpp
@@ -319,7 +319,6 @@
static inline double pseudo_hypergeometric_lb_on_p(uint64_t n, uint32_t k, double sampling_rate);
static bool is_power_of_2(uint32_t v);
static uint32_t to_log_2(uint32_t v);
- static uint32_t ceiling_power_of_2(uint32_t n);
static inline uint32_t next_int(uint32_t max_value);
static inline double next_double_exclude_zero();
@@ -390,4 +389,4 @@
#include "var_opt_sketch_impl.hpp"
-#endif // _VAR_OPT_SKETCH_HPP_
\ No newline at end of file
+#endif // _VAR_OPT_SKETCH_HPP_
diff --git a/sampling/include/var_opt_sketch_impl.hpp b/sampling/include/var_opt_sketch_impl.hpp
index 77a065d..413e932 100644
--- a/sampling/include/var_opt_sketch_impl.hpp
+++ b/sampling/include/var_opt_sketch_impl.hpp
@@ -31,6 +31,7 @@
#include "bounds_binomial_proportions.hpp"
#include "count_zeros.hpp"
#include "memory_operations.hpp"
+#include "ceiling_power_of_2.hpp"
namespace datasketches {
@@ -1744,17 +1745,6 @@
return r;
}
-template<typename T, typename S, typename A>
-uint32_t var_opt_sketch<T,S,A>::ceiling_power_of_2(uint32_t n) {
- --n;
- n |= n >> 1;
- n |= n >> 2;
- n |= n >> 4;
- n |= n >> 8;
- n |= n >> 16;
- return ++n;
-}
-
}
// namespace datasketches
diff --git a/theta/CMakeLists.txt b/theta/CMakeLists.txt
index 69af9db..a68b70b 100644
--- a/theta/CMakeLists.txt
+++ b/theta/CMakeLists.txt
@@ -34,9 +34,8 @@
set(theta_HEADERS "")
list(APPEND theta_HEADERS "include/theta_sketch.hpp;include/theta_union.hpp;include/theta_intersection.hpp")
-list(APPEND theta_HEADERS "include/theta_a_not_b.hpp;include/binomial_bounds.hpp;include/theta_sketch_impl.hpp")
+list(APPEND theta_HEADERS "include/theta_a_not_b.hpp;include/theta_sketch_impl.hpp")
list(APPEND theta_HEADERS "include/theta_union_impl.hpp;include/theta_intersection_impl.hpp;include/theta_a_not_b_impl.hpp")
-list(APPEND theta_HEADERS "include/conditional_back_inserter.hpp")
install(TARGETS theta
EXPORT ${PROJECT_NAME}
@@ -51,10 +50,8 @@
${CMAKE_CURRENT_SOURCE_DIR}/include/theta_union.hpp
${CMAKE_CURRENT_SOURCE_DIR}/include/theta_intersection.hpp
${CMAKE_CURRENT_SOURCE_DIR}/include/theta_a_not_b.hpp
- ${CMAKE_CURRENT_SOURCE_DIR}/include/binomial_bounds.hpp
${CMAKE_CURRENT_SOURCE_DIR}/include/theta_sketch_impl.hpp
${CMAKE_CURRENT_SOURCE_DIR}/include/theta_union_impl.hpp
${CMAKE_CURRENT_SOURCE_DIR}/include/theta_intersection_impl.hpp
${CMAKE_CURRENT_SOURCE_DIR}/include/theta_a_not_b_impl.hpp
- ${CMAKE_CURRENT_SOURCE_DIR}/include/conditional_back_inserter.hpp
)
diff --git a/theta/include/theta_intersection.hpp b/theta/include/theta_intersection.hpp
index f74ae5e..5945c52 100644
--- a/theta/include/theta_intersection.hpp
+++ b/theta/include/theta_intersection.hpp
@@ -57,7 +57,7 @@
* If update() was not called, the state is the infinite "universe",
* which is considered an undefined state, and throws an exception.
* @param ordered optional flag to specify if ordered sketch should be produced
- * @returnthe result of the intersection
+ * @return the result of the intersection
*/
compact_theta_sketch_alloc<A> get_result(bool ordered = true) const;
diff --git a/theta/include/theta_sketch.hpp b/theta/include/theta_sketch.hpp
index 637fb38..b809f71 100644
--- a/theta/include/theta_sketch.hpp
+++ b/theta/include/theta_sketch.hpp
@@ -526,16 +526,6 @@
typedef update_theta_sketch_alloc<std::allocator<void>> update_theta_sketch;
typedef compact_theta_sketch_alloc<std::allocator<void>> compact_theta_sketch;
-// common helping functions
-
-constexpr uint8_t log2(uint32_t n) {
- return (n > 1) ? 1 + log2(n >> 1) : 0;
-}
-
-constexpr uint8_t lg_size_from_count(uint32_t n, double load_factor) {
- return log2(n) + ((n > static_cast<uint32_t>((1 << (log2(n) + 1)) * load_factor)) ? 2 : 1);
-}
-
} /* namespace datasketches */
#include "theta_sketch_impl.hpp"
diff --git a/tuple/CMakeLists.txt b/tuple/CMakeLists.txt
new file mode 100644
index 0000000..d354850
--- /dev/null
+++ b/tuple/CMakeLists.txt
@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+add_library(tuple INTERFACE)
+
+add_library(${PROJECT_NAME}::TUPLE ALIAS tuple)
+
+if (BUILD_TESTS)
+ add_subdirectory(test)
+endif()
+
+target_include_directories(tuple
+ INTERFACE
+ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+ $<INSTALL_INTERFACE:$<INSTALL_PREFIX>/include>
+)
+
+target_link_libraries(tuple INTERFACE common)
+target_compile_features(tuple INTERFACE cxx_std_11)
+
+set(tuple_HEADERS "")
+list(APPEND tuple_HEADERS "include/tuple_sketch.hpp;include/tuple_sketch_impl.hpp")
+list(APPEND tuple_HEADERS "include/tuple_union.hpp;include/tuple_union_impl.hpp")
+list(APPEND tuple_HEADERS "include/tuple_intersection.hpp;include/tuple_intersection_impl.hpp")
+list(APPEND tuple_HEADERS "include/tuple_a_not_b.hpp;include/tuple_a_not_b_impl.hpp")
+list(APPEND tuple_HEADERS "include/array_of_doubles_sketch.hpp;include/array_of_doubles_sketch_impl.hpp")
+list(APPEND tuple_HEADERS "include/array_of_doubles_union.hpp;include/array_of_doubles_union_impl.hpp")
+list(APPEND tuple_HEADERS "include/array_of_doubles_intersection.hpp;include/array_of_doubles_intersection_impl.hpp")
+list(APPEND tuple_HEADERS "include/array_of_doubles_a_not_b.hpp;include/array_of_doubles_a_not_b_impl.hpp")
+list(APPEND tuple_HEADERS "include/theta_update_sketch_base.hpp;include/theta_update_sketch_base_impl.hpp")
+list(APPEND tuple_HEADERS "include/theta_union_base.hpp;include/theta_union_base_impl.hpp")
+list(APPEND tuple_HEADERS "include/theta_intersection_base.hpp;include/theta_intersection_base_impl.hpp")
+list(APPEND tuple_HEADERS "include/theta_set_difference_base.hpp;include/theta_set_difference_base_impl.hpp")
+list(APPEND tuple_HEADERS "include/theta_sketch_experimental.hpp;include/theta_sketch_experimental_impl.hpp")
+list(APPEND tuple_HEADERS "include/theta_union_experimental.hpp;include/theta_union_experimental_impl.hpp")
+list(APPEND tuple_HEADERS "include/theta_intersection_experimental.hpp;include/theta_intersection_experimental_impl.hpp")
+list(APPEND tuple_HEADERS "include/theta_a_not_b_experimental.hpp;include/theta_a_not_b_experimental_impl.hpp")
+list(APPEND tuple_HEADERS "include/bounds_on_ratios_in_sampled_sets.hpp")
+list(APPEND tuple_HEADERS "include/bounds_on_ratios_in_theta_sketched_sets.hpp")
+list(APPEND tuple_HEADERS "include/jaccard_similarity.hpp")
+list(APPEND tuple_HEADERS "include/theta_comparators.hpp")
+list(APPEND tuple_HEADERS "include/theta_cnstants.hpp")
+
+install(TARGETS tuple
+ EXPORT ${PROJECT_NAME}
+)
+
+install(FILES ${tuple_HEADERS}
+ DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches")
+
+target_sources(tuple
+ INTERFACE
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_sketch.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_sketch_impl.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_union.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_union_impl.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_intersection.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_intersection_impl.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_a_not_b.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/tuple_a_not_b_impl.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/array_of_doubles_sketch.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/array_of_doubles_sketch_impl.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/array_of_doubles_union.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/array_of_doubles_union_impl.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/array_of_doubles_intersection.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/array_of_doubles_intersection_impl.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/array_of_doubles_a_not_b.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/array_of_doubles_a_not_b_impl.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_update_sketch_base.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_update_sketch_base_impl.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_union_base.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_union_base_impl.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_intersection_base.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_intersection_base_impl.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_set_difference_base.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_set_difference_base_impl.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_sketch_experimental.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_sketch_experimental_impl.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_union_experimental.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_union_experimental_impl.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_intersection_experimental.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_intersection_experimental_impl.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_a_not_b_experimental.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_a_not_b_experimental_impl.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/bounds_on_ratios_in_sampled_sets.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/bounds_on_ratios_in_theta_sketched_sets.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/jaccard_similarity.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_comparators.hpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/include/theta_constants.hpp
+)
diff --git a/tuple/include/array_of_doubles_a_not_b.hpp b/tuple/include/array_of_doubles_a_not_b.hpp
new file mode 100644
index 0000000..c2bbc4e
--- /dev/null
+++ b/tuple/include/array_of_doubles_a_not_b.hpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef ARRAY_OF_DOUBLES_A_NOT_B_HPP_
+#define ARRAY_OF_DOUBLES_A_NOT_B_HPP_
+
+#include <vector>
+#include <memory>
+
+#include "array_of_doubles_sketch.hpp"
+#include "tuple_a_not_b.hpp"
+
+namespace datasketches {
+
+template<typename Allocator = std::allocator<double>>
+class array_of_doubles_a_not_b_alloc: tuple_a_not_b<std::vector<double, Allocator>, AllocVectorDouble<Allocator>> {
+public:
+ using Summary = std::vector<double, Allocator>;
+ using AllocSummary = AllocVectorDouble<Allocator>;
+ using Base = tuple_a_not_b<Summary, AllocSummary>;
+ using CompactSketch = compact_array_of_doubles_sketch_alloc<Allocator>;
+
+ explicit array_of_doubles_a_not_b_alloc(uint64_t seed = DEFAULT_SEED, const Allocator& allocator = Allocator());
+
+ template<typename FwdSketch, typename Sketch>
+ CompactSketch compute(FwdSketch&& a, const Sketch& b, bool ordered = true) const;
+};
+
+// alias with the default allocator for convenience
+using array_of_doubles_a_not_b = array_of_doubles_a_not_b_alloc<>;
+
+} /* namespace datasketches */
+
+#include "array_of_doubles_a_not_b_impl.hpp"
+
+#endif
diff --git a/tuple/include/array_of_doubles_a_not_b_impl.hpp b/tuple/include/array_of_doubles_a_not_b_impl.hpp
new file mode 100644
index 0000000..d0330e2
--- /dev/null
+++ b/tuple/include/array_of_doubles_a_not_b_impl.hpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+namespace datasketches {
+
+template<typename A>
+array_of_doubles_a_not_b_alloc<A>::array_of_doubles_a_not_b_alloc(uint64_t seed, const A& allocator):
+Base(seed, allocator) {}
+
+template<typename A>
+template<typename FwdSketch, typename Sketch>
+auto array_of_doubles_a_not_b_alloc<A>::compute(FwdSketch&& a, const Sketch& b, bool ordered) const -> CompactSketch {
+ return CompactSketch(a.get_num_values(), Base::compute(std::forward<FwdSketch>(a), b, ordered));
+}
+
+} /* namespace datasketches */
diff --git a/tuple/include/array_of_doubles_intersection.hpp b/tuple/include/array_of_doubles_intersection.hpp
new file mode 100644
index 0000000..89459c3
--- /dev/null
+++ b/tuple/include/array_of_doubles_intersection.hpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef ARRAY_OF_DOUBLES_INTERSECTION_HPP_
+#define ARRAY_OF_DOUBLES_INTERSECTION_HPP_
+
+#include <vector>
+#include <memory>
+
+#include "array_of_doubles_sketch.hpp"
+#include "tuple_intersection.hpp"
+
+namespace datasketches {
+
+template<
+ typename Policy,
+ typename Allocator = std::allocator<double>
+>
+class array_of_doubles_intersection: public tuple_intersection<aod<Allocator>, Policy, AllocAOD<Allocator>> {
+public:
+ using Summary = aod<Allocator>;
+ using AllocSummary = AllocAOD<Allocator>;
+ using Base = tuple_intersection<Summary, Policy, AllocSummary>;
+ using CompactSketch = compact_array_of_doubles_sketch_alloc<Allocator>;
+ using resize_factor = theta_constants::resize_factor;
+
+ explicit array_of_doubles_intersection(uint64_t seed = DEFAULT_SEED, const Policy& policy = Policy(), const Allocator& allocator = Allocator());
+
+ CompactSketch get_result(bool ordered = true) const;
+};
+
+} /* namespace datasketches */
+
+#include "array_of_doubles_intersection_impl.hpp"
+
+#endif
diff --git a/tuple/include/array_of_doubles_intersection_impl.hpp b/tuple/include/array_of_doubles_intersection_impl.hpp
new file mode 100644
index 0000000..7cd2472
--- /dev/null
+++ b/tuple/include/array_of_doubles_intersection_impl.hpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+namespace datasketches {
+
+template<typename P, typename A>
+array_of_doubles_intersection<P, A>::array_of_doubles_intersection(uint64_t seed, const P& policy, const A& allocator):
+Base(seed, policy, allocator) {}
+
+template<typename P, typename A>
+auto array_of_doubles_intersection<P, A>::get_result(bool ordered) const -> CompactSketch {
+ return compact_array_of_doubles_sketch_alloc<A>(this->state_.get_policy().get_policy().get_num_values(), Base::get_result(ordered));
+}
+
+} /* namespace datasketches */
diff --git a/tuple/include/array_of_doubles_sketch.hpp b/tuple/include/array_of_doubles_sketch.hpp
new file mode 100644
index 0000000..af9e87a
--- /dev/null
+++ b/tuple/include/array_of_doubles_sketch.hpp
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef ARRAY_OF_DOUBLES_SKETCH_HPP_
+#define ARRAY_OF_DOUBLES_SKETCH_HPP_
+
+#include <vector>
+#include <memory>
+
+#include "serde.hpp"
+#include "tuple_sketch.hpp"
+
+namespace datasketches {
+
+// This sketch is equivalent of ArrayOfDoublesSketch in Java
+
+// This simple array of double is faster than std::vector and should be sufficient for this application
+template<typename Allocator = std::allocator<double>>
+class aod {
+public:
+ explicit aod(uint8_t size, const Allocator& allocator = Allocator()):
+ allocator_(allocator), size_(size), array_(allocator_.allocate(size_)) {
+ std::fill(array_, array_ + size_, 0);
+ }
+ aod(const aod& other):
+ allocator_(other.allocator_),
+ size_(other.size_),
+ array_(allocator_.allocate(size_))
+ {
+ std::copy(other.array_, other.array_ + size_, array_);
+ }
+ aod(aod&& other) noexcept:
+ allocator_(std::move(other.allocator_)),
+ size_(other.size_),
+ array_(other.array_)
+ {
+ other.array_ = nullptr;
+ }
+ ~aod() {
+ if (array_ != nullptr) allocator_.deallocate(array_, size_);
+ }
+ aod& operator=(const aod& other) {
+ aod copy(other);
+ std::swap(allocator_, copy.allocator_);
+ std::swap(size_, copy.size_);
+ std::swap(array_, copy.array_);
+ return *this;
+ }
+ aod& operator=(aod&& other) {
+ std::swap(allocator_, other.allocator_);
+ std::swap(size_, other.size_);
+ std::swap(array_, other.array_);
+ return *this;
+ }
+ double& operator[](size_t index) { return array_[index]; }
+ double operator[](size_t index) const { return array_[index]; }
+ uint8_t size() const { return size_; }
+ double* data() { return array_; }
+ const double* data() const { return array_; }
+ bool operator==(const aod& other) const {
+ for (uint8_t i = 0; i < size_; ++i) if (array_[i] != other.array_[i]) return false;
+ return true;
+ }
+private:
+ Allocator allocator_;
+ uint8_t size_;
+ double* array_;
+};
+
+template<typename A = std::allocator<double>>
+class array_of_doubles_update_policy {
+public:
+ array_of_doubles_update_policy(uint8_t num_values = 1, const A& allocator = A()):
+ allocator_(allocator), num_values_(num_values) {}
+ aod<A> create() const {
+ return aod<A>(num_values_, allocator_);
+ }
+ template<typename InputVector> // to allow any type with indexed access (such as double*)
+ void update(aod<A>& summary, const InputVector& update) const {
+ for (uint8_t i = 0; i < num_values_; ++i) summary[i] += update[i];
+ }
+ uint8_t get_num_values() const {
+ return num_values_;
+ }
+
+private:
+ A allocator_;
+ uint8_t num_values_;
+};
+
+// forward declaration
+template<typename A> class compact_array_of_doubles_sketch_alloc;
+
+template<typename A> using AllocAOD = typename std::allocator_traits<A>::template rebind_alloc<aod<A>>;
+
+template<typename A = std::allocator<double>>
+class update_array_of_doubles_sketch_alloc: public update_tuple_sketch<aod<A>, aod<A>, array_of_doubles_update_policy<A>, AllocAOD<A>> {
+public:
+ using Base = update_tuple_sketch<aod<A>, aod<A>, array_of_doubles_update_policy<A>, AllocAOD<A>>;
+ using resize_factor = typename Base::resize_factor;
+
+ class builder;
+
+ compact_array_of_doubles_sketch_alloc<A> compact(bool ordered = true) const;
+ uint8_t get_num_values() const;
+
+private:
+ // for builder
+ update_array_of_doubles_sketch_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta,
+ uint64_t seed, const array_of_doubles_update_policy<A>& policy, const A& allocator);
+};
+
+// alias with the default allocator for convenience
+using update_array_of_doubles_sketch = update_array_of_doubles_sketch_alloc<>;
+
+template<typename A>
+class update_array_of_doubles_sketch_alloc<A>::builder: public tuple_base_builder<builder, array_of_doubles_update_policy<A>, A> {
+public:
+ builder(const array_of_doubles_update_policy<A>& policy = array_of_doubles_update_policy<A>(), const A& allocator = A());
+ update_array_of_doubles_sketch_alloc<A> build() const;
+};
+
+template<typename A = std::allocator<double>>
+class compact_array_of_doubles_sketch_alloc: public compact_tuple_sketch<aod<A>, AllocAOD<A>> {
+public:
+ using Base = compact_tuple_sketch<aod<A>, AllocAOD<A>>;
+ using Entry = typename Base::Entry;
+ using AllocEntry = typename Base::AllocEntry;
+ using AllocU64 = typename Base::AllocU64;
+ using vector_bytes = typename Base::vector_bytes;
+
+ static const uint8_t SERIAL_VERSION = 1;
+ static const uint8_t SKETCH_FAMILY = 9;
+ static const uint8_t SKETCH_TYPE = 3;
+ enum flags { UNUSED1, UNUSED2, IS_EMPTY, HAS_ENTRIES, IS_ORDERED };
+
+ template<typename Sketch>
+ compact_array_of_doubles_sketch_alloc(const Sketch& other, bool ordered = true);
+
+ uint8_t get_num_values() const;
+
+ void serialize(std::ostream& os) const;
+ vector_bytes serialize(unsigned header_size_bytes = 0) const;
+
+ static compact_array_of_doubles_sketch_alloc deserialize(std::istream& is, uint64_t seed = DEFAULT_SEED, const A& allocator = A());
+ static compact_array_of_doubles_sketch_alloc deserialize(const void* bytes, size_t size, uint64_t seed = DEFAULT_SEED,
+ const A& allocator = A());
+
+ // for internal use
+ compact_array_of_doubles_sketch_alloc(bool is_empty, bool is_ordered, uint16_t seed_hash, uint64_t theta, std::vector<Entry, AllocEntry>&& entries, uint8_t num_values);
+ compact_array_of_doubles_sketch_alloc(uint8_t num_values, Base&& base);
+private:
+ uint8_t num_values_;
+};
+
+// alias with the default allocator for convenience
+using compact_array_of_doubles_sketch = compact_array_of_doubles_sketch_alloc<>;
+
+} /* namespace datasketches */
+
+#include "array_of_doubles_sketch_impl.hpp"
+
+#endif
diff --git a/tuple/include/array_of_doubles_sketch_impl.hpp b/tuple/include/array_of_doubles_sketch_impl.hpp
new file mode 100644
index 0000000..6457072
--- /dev/null
+++ b/tuple/include/array_of_doubles_sketch_impl.hpp
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+namespace datasketches {
+
+template<typename A>
+update_array_of_doubles_sketch_alloc<A>::update_array_of_doubles_sketch_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf,
+ uint64_t theta, uint64_t seed, const array_of_doubles_update_policy<A>& policy, const A& allocator):
+Base(lg_cur_size, lg_nom_size, rf, theta, seed, policy, allocator) {}
+
+
+template<typename A>
+uint8_t update_array_of_doubles_sketch_alloc<A>::get_num_values() const {
+ return this->policy_.get_num_values();
+}
+
+template<typename A>
+compact_array_of_doubles_sketch_alloc<A> update_array_of_doubles_sketch_alloc<A>::compact(bool ordered) const {
+ return compact_array_of_doubles_sketch_alloc<A>(*this, ordered);
+}
+
+// builder
+
+template<typename A>
+update_array_of_doubles_sketch_alloc<A>::builder::builder(const array_of_doubles_update_policy<A>& policy, const A& allocator):
+tuple_base_builder<builder, array_of_doubles_update_policy<A>, A>(policy, allocator) {}
+
+template<typename A>
+update_array_of_doubles_sketch_alloc<A> update_array_of_doubles_sketch_alloc<A>::builder::build() const {
+ return update_array_of_doubles_sketch_alloc<A>(this->starting_lg_size(), this->lg_k_, this->rf_, this->starting_theta(), this->seed_, this->policy_, this->allocator_);
+}
+
+// compact sketch
+
+template<typename A>
+template<typename S>
+compact_array_of_doubles_sketch_alloc<A>::compact_array_of_doubles_sketch_alloc(const S& other, bool ordered):
+Base(other, ordered), num_values_(other.get_num_values()) {}
+
+template<typename A>
+compact_array_of_doubles_sketch_alloc<A>::compact_array_of_doubles_sketch_alloc(bool is_empty, bool is_ordered,
+ uint16_t seed_hash, uint64_t theta, std::vector<Entry, AllocEntry>&& entries, uint8_t num_values):
+Base(is_empty, is_ordered, seed_hash, theta, std::move(entries)), num_values_(num_values) {}
+
+template<typename A>
+compact_array_of_doubles_sketch_alloc<A>::compact_array_of_doubles_sketch_alloc(uint8_t num_values, Base&& base):
+Base(std::move(base)), num_values_(num_values) {}
+
+template<typename A>
+uint8_t compact_array_of_doubles_sketch_alloc<A>::get_num_values() const {
+ return num_values_;
+}
+
+template<typename A>
+void compact_array_of_doubles_sketch_alloc<A>::serialize(std::ostream& os) const {
+ const uint8_t preamble_longs = 1;
+ os.write(reinterpret_cast<const char*>(&preamble_longs), sizeof(preamble_longs));
+ const uint8_t serial_version = SERIAL_VERSION;
+ os.write(reinterpret_cast<const char*>(&serial_version), sizeof(serial_version));
+ const uint8_t family = SKETCH_FAMILY;
+ os.write(reinterpret_cast<const char*>(&family), sizeof(family));
+ const uint8_t type = SKETCH_TYPE;
+ os.write(reinterpret_cast<const char*>(&type), sizeof(type));
+ const uint8_t flags_byte(
+ (this->is_empty() ? 1 << flags::IS_EMPTY : 0) |
+ (this->get_num_retained() > 0 ? 1 << flags::HAS_ENTRIES : 0) |
+ (this->is_ordered() ? 1 << flags::IS_ORDERED : 0)
+ );
+ os.write(reinterpret_cast<const char*>(&flags_byte), sizeof(flags_byte));
+ os.write(reinterpret_cast<const char*>(&num_values_), sizeof(num_values_));
+ const uint16_t seed_hash = this->get_seed_hash();
+ os.write(reinterpret_cast<const char*>(&seed_hash), sizeof(seed_hash));
+ os.write(reinterpret_cast<const char*>(&(this->theta_)), sizeof(uint64_t));
+ if (this->get_num_retained() > 0) {
+ const uint32_t num_entries = this->entries_.size();
+ os.write(reinterpret_cast<const char*>(&num_entries), sizeof(num_entries));
+ const uint32_t unused32 = 0;
+ os.write(reinterpret_cast<const char*>(&unused32), sizeof(unused32));
+ for (const auto& it: this->entries_) {
+ os.write(reinterpret_cast<const char*>(&it.first), sizeof(uint64_t));
+ }
+ for (const auto& it: this->entries_) {
+ os.write(reinterpret_cast<const char*>(it.second.data()), it.second.size() * sizeof(double));
+ }
+ }
+}
+
+template<typename A>
+auto compact_array_of_doubles_sketch_alloc<A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
+ const uint8_t preamble_longs = 1;
+ const size_t size = header_size_bytes + 16 // preamble and theta
+ + (this->entries_.size() > 0 ? 8 : 0)
+ + (sizeof(uint64_t) + sizeof(double) * num_values_) * this->entries_.size();
+ vector_bytes bytes(size, 0, this->entries_.get_allocator());
+ uint8_t* ptr = bytes.data() + header_size_bytes;
+
+ ptr += copy_to_mem(&preamble_longs, ptr, sizeof(preamble_longs));
+ const uint8_t serial_version = SERIAL_VERSION;
+ ptr += copy_to_mem(&serial_version, ptr, sizeof(serial_version));
+ const uint8_t family = SKETCH_FAMILY;
+ ptr += copy_to_mem(&family, ptr, sizeof(family));
+ const uint8_t type = SKETCH_TYPE;
+ ptr += copy_to_mem(&type, ptr, sizeof(type));
+ const uint8_t flags_byte(
+ (this->is_empty() ? 1 << flags::IS_EMPTY : 0) |
+ (this->get_num_retained() ? 1 << flags::HAS_ENTRIES : 0) |
+ (this->is_ordered() ? 1 << flags::IS_ORDERED : 0)
+ );
+ ptr += copy_to_mem(&flags_byte, ptr, sizeof(flags_byte));
+ ptr += copy_to_mem(&num_values_, ptr, sizeof(num_values_));
+ const uint16_t seed_hash = this->get_seed_hash();
+ ptr += copy_to_mem(&seed_hash, ptr, sizeof(seed_hash));
+ ptr += copy_to_mem(&(this->theta_), ptr, sizeof(uint64_t));
+ if (this->get_num_retained() > 0) {
+ const uint32_t num_entries = this->entries_.size();
+ ptr += copy_to_mem(&num_entries, ptr, sizeof(num_entries));
+ const uint32_t unused32 = 0;
+ ptr += copy_to_mem(&unused32, ptr, sizeof(unused32));
+ for (const auto& it: this->entries_) {
+ ptr += copy_to_mem(&it.first, ptr, sizeof(uint64_t));
+ }
+ for (const auto& it: this->entries_) {
+ ptr += copy_to_mem(it.second.data(), ptr, it.second.size() * sizeof(double));
+ }
+ }
+ return bytes;
+}
+
+template<typename A>
+compact_array_of_doubles_sketch_alloc<A> compact_array_of_doubles_sketch_alloc<A>::deserialize(std::istream& is, uint64_t seed, const A& allocator) {
+ uint8_t preamble_longs;
+ is.read(reinterpret_cast<char*>(&preamble_longs), sizeof(preamble_longs));
+ uint8_t serial_version;
+ is.read(reinterpret_cast<char*>(&serial_version), sizeof(serial_version));
+ uint8_t family;
+ is.read(reinterpret_cast<char*>(&family), sizeof(family));
+ uint8_t type;
+ is.read(reinterpret_cast<char*>(&type), sizeof(type));
+ uint8_t flags_byte;
+ is.read(reinterpret_cast<char*>(&flags_byte), sizeof(flags_byte));
+ uint8_t num_values;
+ is.read(reinterpret_cast<char*>(&num_values), sizeof(num_values));
+ uint16_t seed_hash;
+ is.read(reinterpret_cast<char*>(&seed_hash), sizeof(seed_hash));
+ checker<true>::check_serial_version(serial_version, SERIAL_VERSION);
+ checker<true>::check_sketch_family(family, SKETCH_FAMILY);
+ checker<true>::check_sketch_type(type, SKETCH_TYPE);
+ const bool has_entries = flags_byte & (1 << flags::HAS_ENTRIES);
+ if (has_entries) checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+
+ uint64_t theta;
+ is.read(reinterpret_cast<char*>(&theta), sizeof(theta));
+ std::vector<Entry, AllocEntry> entries(allocator);
+ if (has_entries) {
+ uint32_t num_entries;
+ is.read(reinterpret_cast<char*>(&num_entries), sizeof(num_entries));
+ uint32_t unused32;
+ is.read(reinterpret_cast<char*>(&unused32), sizeof(unused32));
+ entries.reserve(num_entries);
+ std::vector<uint64_t, AllocU64> keys(num_entries, 0, allocator);
+ is.read(reinterpret_cast<char*>(keys.data()), num_entries * sizeof(uint64_t));
+ for (size_t i = 0; i < num_entries; ++i) {
+ aod<A> summary(num_values, allocator);
+ is.read(reinterpret_cast<char*>(summary.data()), num_values * sizeof(double));
+ entries.push_back(Entry(keys[i], std::move(summary)));
+ }
+ }
+ if (!is.good()) throw std::runtime_error("error reading from std::istream");
+ const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
+ const bool is_ordered = flags_byte & (1 << flags::IS_ORDERED);
+ return compact_array_of_doubles_sketch_alloc(is_empty, is_ordered, seed_hash, theta, std::move(entries), num_values);
+}
+
+template<typename A>
+compact_array_of_doubles_sketch_alloc<A> compact_array_of_doubles_sketch_alloc<A>::deserialize(const void* bytes, size_t size, uint64_t seed, const A& allocator) {
+ ensure_minimum_memory(size, 16);
+ const char* ptr = static_cast<const char*>(bytes);
+ uint8_t preamble_longs;
+ ptr += copy_from_mem(ptr, &preamble_longs, sizeof(preamble_longs));
+ uint8_t serial_version;
+ ptr += copy_from_mem(ptr, &serial_version, sizeof(serial_version));
+ uint8_t family;
+ ptr += copy_from_mem(ptr, &family, sizeof(family));
+ uint8_t type;
+ ptr += copy_from_mem(ptr, &type, sizeof(type));
+ uint8_t flags_byte;
+ ptr += copy_from_mem(ptr, &flags_byte, sizeof(flags_byte));
+ uint8_t num_values;
+ ptr += copy_from_mem(ptr, &num_values, sizeof(num_values));
+ uint16_t seed_hash;
+ ptr += copy_from_mem(ptr, &seed_hash, sizeof(seed_hash));
+ checker<true>::check_serial_version(serial_version, SERIAL_VERSION);
+ checker<true>::check_sketch_family(family, SKETCH_FAMILY);
+ checker<true>::check_sketch_type(type, SKETCH_TYPE);
+ const bool has_entries = flags_byte & (1 << flags::HAS_ENTRIES);
+ if (has_entries) checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+
+ uint64_t theta;
+ ptr += copy_from_mem(ptr, &theta, sizeof(theta));
+ std::vector<Entry, AllocEntry> entries(allocator);
+ if (has_entries) {
+ ensure_minimum_memory(size, 24);
+ uint32_t num_entries;
+ ptr += copy_from_mem(ptr, &num_entries, sizeof(num_entries));
+ uint32_t unused32;
+ ptr += copy_from_mem(ptr, &unused32, sizeof(unused32));
+ ensure_minimum_memory(size, 24 + (sizeof(uint64_t) + sizeof(double) * num_values) * num_entries);
+ entries.reserve(num_entries);
+ std::vector<uint64_t, AllocU64> keys(num_entries, 0, allocator);
+ ptr += copy_from_mem(ptr, keys.data(), sizeof(uint64_t) * num_entries);
+ for (size_t i = 0; i < num_entries; ++i) {
+ aod<A> summary(num_values, allocator);
+ ptr += copy_from_mem(ptr, summary.data(), num_values * sizeof(double));
+ entries.push_back(Entry(keys[i], std::move(summary)));
+ }
+ }
+ const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
+ const bool is_ordered = flags_byte & (1 << flags::IS_ORDERED);
+ return compact_array_of_doubles_sketch_alloc(is_empty, is_ordered, seed_hash, theta, std::move(entries), num_values);
+}
+
+} /* namespace datasketches */
diff --git a/tuple/include/array_of_doubles_union.hpp b/tuple/include/array_of_doubles_union.hpp
new file mode 100644
index 0000000..592c1f9
--- /dev/null
+++ b/tuple/include/array_of_doubles_union.hpp
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef ARRAY_OF_DOUBLES_UNION_HPP_
+#define ARRAY_OF_DOUBLES_UNION_HPP_
+
+#include <vector>
+#include <memory>
+
+#include "array_of_doubles_sketch.hpp"
+#include "tuple_union.hpp"
+
+namespace datasketches {
+
+template<typename A = std::allocator<double>>
+struct array_of_doubles_union_policy_alloc {
+ array_of_doubles_union_policy_alloc(uint8_t num_values = 1): num_values_(num_values) {}
+
+ void operator()(aod<A>& summary, const aod<A>& other) const {
+ for (size_t i = 0; i < summary.size(); ++i) {
+ summary[i] += other[i];
+ }
+ }
+
+ uint8_t get_num_values() const {
+ return num_values_;
+ }
+private:
+ uint8_t num_values_;
+};
+
+using array_of_doubles_union_policy = array_of_doubles_union_policy_alloc<>;
+
+template<typename Allocator = std::allocator<double>>
+class array_of_doubles_union_alloc: public tuple_union<aod<Allocator>, array_of_doubles_union_policy_alloc<Allocator>, AllocAOD<Allocator>> {
+public:
+ using Policy = array_of_doubles_union_policy_alloc<Allocator>;
+ using Base = tuple_union<aod<Allocator>, Policy, AllocAOD<Allocator>>;
+ using CompactSketch = compact_array_of_doubles_sketch_alloc<Allocator>;
+ using resize_factor = theta_constants::resize_factor;
+
+ class builder;
+
+ CompactSketch get_result(bool ordered = true) const;
+
+private:
+ // for builder
+ array_of_doubles_union_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta, uint64_t seed, const Policy& policy, const Allocator& allocator);
+};
+
+template<typename Allocator>
+class array_of_doubles_union_alloc<Allocator>::builder: public tuple_base_builder<builder, array_of_doubles_union_policy_alloc<Allocator>, Allocator> {
+public:
+ builder(const array_of_doubles_union_policy_alloc<Allocator>& policy = array_of_doubles_union_policy_alloc<Allocator>(), const Allocator& allocator = Allocator());
+ array_of_doubles_union_alloc<Allocator> build() const;
+};
+
+// alias with default allocator
+using array_of_doubles_union = array_of_doubles_union_alloc<>;
+
+} /* namespace datasketches */
+
+#include "array_of_doubles_union_impl.hpp"
+
+#endif
diff --git a/tuple/include/array_of_doubles_union_impl.hpp b/tuple/include/array_of_doubles_union_impl.hpp
new file mode 100644
index 0000000..57899d9
--- /dev/null
+++ b/tuple/include/array_of_doubles_union_impl.hpp
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+namespace datasketches {
+
+template<typename A>
+array_of_doubles_union_alloc<A>::array_of_doubles_union_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta, uint64_t seed, const Policy& policy, const A& allocator):
+Base(lg_cur_size, lg_nom_size, rf, theta, seed, policy, allocator)
+{}
+
+template<typename A>
+auto array_of_doubles_union_alloc<A>::get_result(bool ordered) const -> CompactSketch {
+ return compact_array_of_doubles_sketch_alloc<A>(this->state_.get_policy().get_policy().get_num_values(), Base::get_result(ordered));
+}
+
+// builder
+
+template<typename A>
+array_of_doubles_union_alloc<A>::builder::builder(const Policy& policy, const A& allocator):
+tuple_base_builder<builder, Policy, A>(policy, allocator) {}
+
+template<typename A>
+array_of_doubles_union_alloc<A> array_of_doubles_union_alloc<A>::builder::build() const {
+ return array_of_doubles_union_alloc<A>(this->starting_lg_size(), this->lg_k_, this->rf_, this->starting_theta(), this->seed_, this->policy_, this->allocator_);
+}
+
+} /* namespace datasketches */
diff --git a/tuple/include/bounds_on_ratios_in_sampled_sets.hpp b/tuple/include/bounds_on_ratios_in_sampled_sets.hpp
new file mode 100644
index 0000000..39accd5
--- /dev/null
+++ b/tuple/include/bounds_on_ratios_in_sampled_sets.hpp
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef BOUNDS_ON_RATIOS_IN_SAMPLED_SETS_HPP_
+#define BOUNDS_ON_RATIOS_IN_SAMPLED_SETS_HPP_
+
+#include <cstdint>
+
+#include <bounds_binomial_proportions.hpp>
+
+namespace datasketches {
+
+/**
+ * This class is used to compute the bounds on the estimate of the ratio <i>|B| / |A|</i>, where:
+ * <ul>
+ * <li><i>|A|</i> is the unknown size of a set <i>A</i> of unique identifiers.</li>
+ * <li><i>|B|</i> is the unknown size of a subset <i>B</i> of <i>A</i>.</li>
+ * <li><i>a</i> = <i>|S<sub>A</sub>|</i> is the observed size of a sample of <i>A</i>
+ * that was obtained by Bernoulli sampling with a known inclusion probability <i>f</i>.</li>
+ * <li><i>b</i> = <i>|S<sub>A</sub> ∩ B|</i> is the observed size of a subset
+ * of <i>S<sub>A</sub></i>.</li>
+ * </ul>
+ */
+class bounds_on_ratios_in_sampled_sets {
+public:
+ static constexpr double NUM_STD_DEVS = 2.0;
+
+ /**
+ * Return the approximate lower bound based on a 95% confidence interval
+ * @param a See class javadoc
+ * @param b See class javadoc
+ * @param f the inclusion probability used to produce the set with size <i>a</i> and should
+ * generally be less than 0.5. Above this value, the results not be reliable.
+ * When <i>f</i> = 1.0 this returns the estimate.
+ * @return the approximate upper bound
+ */
+ static double lower_bound_for_b_over_a(uint64_t a, uint64_t b, double f) {
+ check_inputs(a, b, f);
+ if (a == 0) return 0.0;
+ if (f == 1.0) return static_cast<double>(b) / static_cast<double>(a);
+ return bounds_binomial_proportions::approximate_lower_bound_on_p(a, b, NUM_STD_DEVS * hacky_adjuster(f));
+ }
+
+ /**
+ * Return the approximate upper bound based on a 95% confidence interval
+ * @param a See class javadoc
+ * @param b See class javadoc
+ * @param f the inclusion probability used to produce the set with size <i>a</i>.
+ * @return the approximate lower bound
+ */
+ static double upper_bound_for_b_over_a(uint64_t a, uint64_t b, double f) {
+ check_inputs(a, b, f);
+ if (a == 0) return 1.0;
+ if (f == 1.0) return static_cast<double>(b) / static_cast<double>(a);
+ return bounds_binomial_proportions::approximate_upper_bound_on_p(a, b, NUM_STD_DEVS * hacky_adjuster(f));
+ }
+
+ /**
+ * Return the estimate of b over a
+ * @param a See class javadoc
+ * @param b See class javadoc
+ * @return the estimate of b over a
+ */
+ static double get_estimate_of_b_over_a(uint64_t a, uint64_t b) {
+ check_inputs(a, b, 0.3);
+ if (a == 0) return 0.5;
+ return static_cast<double>(b) / static_cast<double>(a);
+ }
+
+ /**
+ * Return the estimate of A. See class javadoc.
+ * @param a See class javadoc
+ * @param f the inclusion probability used to produce the set with size <i>a</i>.
+ * @return the approximate lower bound
+ */
+ static double estimate_of_a(uint64_t a, uint64_t f) {
+ check_inputs(a, 1, f);
+ return a / f;
+ }
+
+ /**
+ * Return the estimate of B. See class javadoc.
+ * @param b See class javadoc
+ * @param f the inclusion probability used to produce the set with size <i>b</i>.
+ * @return the approximate lower bound
+ */
+ static double estimate_of_b(uint64_t b, double f) {
+ check_inputs(b + 1, b, f);
+ return b / f;
+ }
+
+private:
+ /**
+ * This hackyAdjuster is tightly coupled with the width of the confidence interval normally
+ * specified with number of standard deviations. To simplify this interface the number of
+ * standard deviations has been fixed to 2.0, which corresponds to a confidence interval of
+ * 95%.
+ * @param f the inclusion probability used to produce the set with size <i>a</i>.
+ * @return the hacky Adjuster
+ */
+ static double hacky_adjuster(double f) {
+ const double tmp = sqrt(1.0 - f);
+ return (f <= 0.5) ? tmp : tmp + (0.01 * (f - 0.5));
+ }
+
+ static void check_inputs(uint64_t a, uint64_t b, double f) {
+ if (a < b) {
+ throw std::invalid_argument("a must be >= b: a = " + std::to_string(a) + ", b = " + std::to_string(b));
+ }
+ if ((f > 1.0) || (f <= 0.0)) {
+ throw std::invalid_argument("Required: ((f <= 1.0) && (f > 0.0)): " + std::to_string(f));
+ }
+ }
+
+};
+
+} /* namespace datasketches */
+
+# endif
diff --git a/tuple/include/bounds_on_ratios_in_theta_sketched_sets.hpp b/tuple/include/bounds_on_ratios_in_theta_sketched_sets.hpp
new file mode 100644
index 0000000..3cf9cb4
--- /dev/null
+++ b/tuple/include/bounds_on_ratios_in_theta_sketched_sets.hpp
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef BOUNDS_ON_RATIOS_IN_THETA_SKETCHED_SETS_HPP_
+#define BOUNDS_ON_RATIOS_IN_THETA_SKETCHED_SETS_HPP_
+
+#include <cstdint>
+#include <stdexcept>
+
+#include <bounds_on_ratios_in_sampled_sets.hpp>
+
+namespace datasketches {
+
+/**
+ * This is to compute the bounds on the estimate of the ratio <i>B / A</i>, where:
+ * <ul>
+ * <li><i>A</i> is a Theta Sketch of population <i>PopA</i>.</li>
+ * <li><i>B</i> is a Theta Sketch of population <i>PopB</i> that is a subset of <i>A</i>,
+ * obtained by an intersection of <i>A</i> with some other Theta Sketch <i>C</i>,
+ * which acts like a predicate or selection clause.</li>
+ * <li>The estimate of the ratio <i>PopB/PopA</i> is
+ * estimate_of_b_over_a(<i>A, B</i>).</li>
+ * <li>The Upper Bound estimate on the ratio PopB/PopA is
+ * upper_bound_for_b_over_a(<i>A, B</i>).</li>
+ * <li>The Lower Bound estimate on the ratio PopB/PopA is
+ * lower_bound_for_b_over_a(<i>A, B</i>).</li>
+ * </ul>
+ * Note: The theta of <i>A</i> cannot be greater than the theta of <i>B</i>.
+ * If <i>B</i> is formed as an intersection of <i>A</i> and some other set <i>C</i>,
+ * then the theta of <i>B</i> is guaranteed to be less than or equal to the theta of <i>B</i>.
+ */
+template<typename ExtractKey>
+class bounds_on_ratios_in_theta_sketched_sets {
+public:
+ /**
+ * Gets the approximate lower bound for B over A based on a 95% confidence interval
+ * @param sketchA the sketch A
+ * @param sketchB the sketch B
+ * @return the approximate lower bound for B over A
+ */
+ template<typename SketchA, typename SketchB>
+ static double lower_bound_for_b_over_a(const SketchA& sketch_a, const SketchB& sketch_b) {
+ const uint64_t theta64_a = sketch_a.get_theta64();
+ const uint64_t theta64_b = sketch_b.get_theta64();
+ check_thetas(theta64_a, theta64_b);
+
+ const uint64_t count_b = sketch_b.get_num_retained();
+ const uint64_t count_a = theta64_a == theta64_b
+ ? sketch_a.get_num_retained()
+ : count_less_than_theta64(sketch_a, theta64_b);
+
+ if (count_a == 0) return 0;
+ const double f = sketch_b.get_theta();
+ return bounds_on_ratios_in_sampled_sets::lower_bound_for_b_over_a(count_a, count_b, f);
+ }
+
+ /**
+ * Gets the approximate upper bound for B over A based on a 95% confidence interval
+ * @param sketchA the sketch A
+ * @param sketchB the sketch B
+ * @return the approximate upper bound for B over A
+ */
+ template<typename SketchA, typename SketchB>
+ static double upper_bound_for_b_over_a(const SketchA& sketch_a, const SketchB& sketch_b) {
+ const uint64_t theta64_a = sketch_a.get_theta64();
+ const uint64_t theta64_b = sketch_b.get_theta64();
+ check_thetas(theta64_a, theta64_b);
+
+ const uint64_t count_b = sketch_b.get_num_retained();
+ const uint64_t count_a = (theta64_a == theta64_b)
+ ? sketch_a.get_num_retained()
+ : count_less_than_theta64(sketch_a, theta64_b);
+
+ if (count_a == 0) return 1;
+ const double f = sketch_b.get_theta();
+ return bounds_on_ratios_in_sampled_sets::upper_bound_for_b_over_a(count_a, count_b, f);
+ }
+
+ /**
+ * Gets the estimate for B over A
+ * @param sketchA the sketch A
+ * @param sketchB the sketch B
+ * @return the estimate for B over A
+ */
+ template<typename SketchA, typename SketchB>
+ static double estimate_of_b_over_a(const SketchA& sketch_a, const SketchB& sketch_b) {
+ const uint64_t theta64_a = sketch_a.get_theta64();
+ const uint64_t theta64_b = sketch_b.get_theta64();
+ check_thetas(theta64_a, theta64_b);
+
+ const uint64_t count_b = sketch_b.get_num_retained();
+ const uint64_t count_a = (theta64_a == theta64_b)
+ ? sketch_a.get_num_retained()
+ : count_less_than_theta64(sketch_a, theta64_b);
+
+ if (count_a == 0) return 0.5;
+ return static_cast<double>(count_b) / static_cast<double>(count_a);
+ }
+
+private:
+
+ static inline void check_thetas(uint64_t theta_a, uint64_t theta_b) {
+ if (theta_b > theta_a) {
+ throw std::invalid_argument("theta_a must be <= theta_b");
+ }
+ }
+
+ template<typename Sketch>
+ static uint64_t count_less_than_theta64(const Sketch& sketch, uint64_t theta) {
+ uint64_t count = 0;
+ for (const auto& entry: sketch) if (ExtractKey()(entry) < theta) ++count;
+ return count;
+ }
+
+};
+
+} /* namespace datasketches */
+
+# endif
diff --git a/tuple/include/jaccard_similarity.hpp b/tuple/include/jaccard_similarity.hpp
new file mode 100644
index 0000000..a77884b
--- /dev/null
+++ b/tuple/include/jaccard_similarity.hpp
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef JACCARD_SIMILARITY_BASE_HPP_
+#define JACCARD_SIMILARITY_BASE_HPP_
+
+#include <memory>
+#include <array>
+
+#include <theta_union_experimental.hpp>
+#include <theta_intersection_experimental.hpp>
+#include <tuple_union.hpp>
+#include <tuple_intersection.hpp>
+#include <bounds_on_ratios_in_theta_sketched_sets.hpp>
+#include <ceiling_power_of_2.hpp>
+#include <common_defs.hpp>
+
+namespace datasketches {
+
+template<typename Union, typename Intersection, typename ExtractKey>
+class jaccard_similarity_base {
+public:
+
+ /**
+ * Computes the Jaccard similarity index with upper and lower bounds. The Jaccard similarity index
+ * <i>J(A,B) = (A ^ B)/(A U B)</i> is used to measure how similar the two sketches are to each
+ * other. If J = 1.0, the sketches are considered equal. If J = 0, the two sketches are
+ * disjoint. A Jaccard of .95 means the overlap between the two
+ * sets is 95% of the union of the two sets.
+ *
+ * <p>Note: For very large pairs of sketches, where the configured nominal entries of the sketches
+ * are 2^25 or 2^26, this method may produce unpredictable results.
+ *
+ * @param sketch_a given sketch A
+ * @param sketch_b given sketch B
+ * @return a double array {LowerBound, Estimate, UpperBound} of the Jaccard index.
+ * The Upper and Lower bounds are for a confidence interval of 95.4% or +/- 2 standard deviations.
+ */
+ template<typename SketchA, typename SketchB>
+ static std::array<double, 3> jaccard(const SketchA& sketch_a, const SketchB& sketch_b) {
+ if (reinterpret_cast<const void*>(&sketch_a) == reinterpret_cast<const void*>(&sketch_b)) return {1, 1, 1};
+ if (sketch_a.is_empty() && sketch_b.is_empty()) return {1, 1, 1};
+ if (sketch_a.is_empty() || sketch_b.is_empty()) return {0, 0, 0};
+
+ auto union_ab = compute_union(sketch_a, sketch_b);
+ if (identical_sets(sketch_a, sketch_b, union_ab)) return {1, 1, 1};
+
+ // intersection
+ Intersection i;
+ i.update(sketch_a);
+ i.update(sketch_b);
+ i.update(union_ab); // ensures that intersection is a subset of the union
+ auto inter_abu = i.get_result(false);
+
+ return {
+ bounds_on_ratios_in_theta_sketched_sets<ExtractKey>::lower_bound_for_b_over_a(union_ab, inter_abu),
+ bounds_on_ratios_in_theta_sketched_sets<ExtractKey>::estimate_of_b_over_a(union_ab, inter_abu),
+ bounds_on_ratios_in_theta_sketched_sets<ExtractKey>::upper_bound_for_b_over_a(union_ab, inter_abu)
+ };
+ }
+
+ /**
+ * Returns true if the two given sketches are equivalent.
+ * @param sketch_a the given sketch A
+ * @param sketch_b the given sketch B
+ * @return true if the two given sketches are exactly equal
+ */
+ template<typename SketchA, typename SketchB>
+ static bool exactly_equal(const SketchA& sketch_a, const SketchB& sketch_b) {
+ if (reinterpret_cast<const void*>(&sketch_a) == reinterpret_cast<const void*>(&sketch_b)) return true;
+ if (sketch_a.is_empty() && sketch_b.is_empty()) return true;
+ if (sketch_a.is_empty() || sketch_b.is_empty()) return false;
+
+ auto union_ab = compute_union(sketch_a, sketch_b);
+ if (identical_sets(sketch_a, sketch_b, union_ab)) return true;
+ return false;
+ }
+
+ /**
+ * Tests similarity of an actual Sketch against an expected Sketch.
+ * Computes the lower bound of the Jaccard index <i>J<sub>LB</sub></i> of the actual and
+ * expected sketches.
+ * if <i>J<sub>LB</sub> ≥ threshold</i>, then the sketches are considered to be
+ * similar with a confidence of 97.7%.
+ *
+ * @param actual the sketch to be tested
+ * @param expected the reference sketch that is considered to be correct
+ * @param threshold a real value between zero and one
+ * @return true if the similarity of the two sketches is greater than the given threshold
+ * with at least 97.7% confidence
+ */
+ template<typename SketchA, typename SketchB>
+ static bool similarity_test(const SketchA& actual, const SketchB& expected, double threshold) {
+ auto jc = jaccard(actual, expected);
+ return jc[0] >= threshold;
+ }
+
+ /**
+ * Tests dissimilarity of an actual Sketch against an expected Sketch.
+ * Computes the upper bound of the Jaccard index <i>J<sub>UB</sub></i> of the actual and
+ * expected sketches.
+ * if <i>J<sub>UB</sub> ≤ threshold</i>, then the sketches are considered to be
+ * dissimilar with a confidence of 97.7%.
+ *
+ * @param actual the sketch to be tested
+ * @param expected the reference sketch that is considered to be correct
+ * @param threshold a real value between zero and one
+ * @return true if the dissimilarity of the two sketches is greater than the given threshold
+ * with at least 97.7% confidence
+ */
+ template<typename SketchA, typename SketchB>
+ static bool dissimilarity_test(const SketchA& actual, const SketchB& expected, double threshold) {
+ auto jc = jaccard(actual, expected);
+ return jc[2] <= threshold;
+ }
+
+private:
+
+ template<typename SketchA, typename SketchB>
+ static typename Union::CompactSketch compute_union(const SketchA& sketch_a, const SketchB& sketch_b) {
+ const unsigned count_a = sketch_a.get_num_retained();
+ const unsigned count_b = sketch_b.get_num_retained();
+ const unsigned lg_k = std::min(std::max(log2(ceiling_power_of_2(count_a + count_b)), theta_constants::MIN_LG_K), theta_constants::MAX_LG_K);
+ auto u = typename Union::builder().set_lg_k(lg_k).build();
+ u.update(sketch_a);
+ u.update(sketch_b);
+ return u.get_result(false);
+ }
+
+ template<typename SketchA, typename SketchB, typename UnionAB>
+ static bool identical_sets(const SketchA& sketch_a, const SketchB& sketch_b, const UnionAB& union_ab) {
+ if (union_ab.get_num_retained() == sketch_a.get_num_retained() &&
+ union_ab.get_num_retained() == sketch_b.get_num_retained() &&
+ union_ab.get_theta64() == sketch_a.get_theta64() &&
+ union_ab.get_theta64() == sketch_b.get_theta64()) return true;
+ return false;
+ }
+
+};
+
+template<typename Allocator>
+using theta_jaccard_similarity_alloc = jaccard_similarity_base<theta_union_experimental<Allocator>, theta_intersection_experimental<Allocator>, trivial_extract_key>;
+
+// alias with default allocator for convenience
+using theta_jaccard_similarity = theta_jaccard_similarity_alloc<std::allocator<uint64_t>>;
+
+template<
+ typename Summary,
+ typename IntersectionPolicy,
+ typename UnionPolicy = default_union_policy<Summary>,
+ typename Allocator = std::allocator<Summary>>
+using tuple_jaccard_similarity = jaccard_similarity_base<tuple_union<Summary, UnionPolicy, Allocator>, tuple_intersection<Summary, IntersectionPolicy, Allocator>, pair_extract_key<uint64_t, Summary>>;
+
+} /* namespace datasketches */
+
+# endif
diff --git a/tuple/include/theta_a_not_b_experimental.hpp b/tuple/include/theta_a_not_b_experimental.hpp
new file mode 100644
index 0000000..ba35dc7
--- /dev/null
+++ b/tuple/include/theta_a_not_b_experimental.hpp
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef THETA_A_NOT_B_EXPERIMENTAL_HPP_
+#define THETA_A_NOT_B_EXPERIMENTAL_HPP_
+
+#include "theta_sketch_experimental.hpp"
+#include "theta_set_difference_base.hpp"
+
+namespace datasketches {
+
+template<typename Allocator = std::allocator<uint64_t>>
+class theta_a_not_b_experimental {
+public:
+ using Entry = uint64_t;
+ using ExtractKey = trivial_extract_key;
+ using CompactSketch = compact_theta_sketch_experimental<Allocator>;
+ using State = theta_set_difference_base<Entry, ExtractKey, CompactSketch, Allocator>;
+
+ explicit theta_a_not_b_experimental(uint64_t seed = DEFAULT_SEED, const Allocator& allocator = Allocator());
+
+ /**
+ * Computes the a-not-b set operation given two sketches.
+ * @return the result of a-not-b
+ */
+ template<typename FwdSketch, typename Sketch>
+ CompactSketch compute(FwdSketch&& a, const Sketch& b, bool ordered = true) const;
+
+private:
+ State state_;
+};
+
+} /* namespace datasketches */
+
+#include "theta_a_not_b_experimental_impl.hpp"
+
+#endif
diff --git a/tuple/include/theta_a_not_b_experimental_impl.hpp b/tuple/include/theta_a_not_b_experimental_impl.hpp
new file mode 100644
index 0000000..1fc321b
--- /dev/null
+++ b/tuple/include/theta_a_not_b_experimental_impl.hpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+namespace datasketches {
+
+template<typename A>
+theta_a_not_b_experimental<A>::theta_a_not_b_experimental(uint64_t seed, const A& allocator):
+state_(seed, allocator)
+{}
+
+template<typename A>
+template<typename FwdSketch, typename Sketch>
+auto theta_a_not_b_experimental<A>::compute(FwdSketch&& a, const Sketch& b, bool ordered) const -> CompactSketch {
+ return state_.compute(std::forward<FwdSketch>(a), b, ordered);
+}
+
+} /* namespace datasketches */
diff --git a/tuple/include/theta_comparators.hpp b/tuple/include/theta_comparators.hpp
new file mode 100644
index 0000000..e8a39b7
--- /dev/null
+++ b/tuple/include/theta_comparators.hpp
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef THETA_COMPARATORS_HPP_
+#define THETA_COMPARATORS_HPP_
+
+namespace datasketches {
+
+template<typename ExtractKey>
+struct compare_by_key {
+ template<typename Entry1, typename Entry2>
+ bool operator()(Entry1&& a, Entry2&& b) const {
+ return ExtractKey()(std::forward<Entry1>(a)) < ExtractKey()(std::forward<Entry2>(b));
+ }
+};
+
+// less than
+
+template<typename Key, typename Entry, typename ExtractKey>
+class key_less_than {
+public:
+ explicit key_less_than(const Key& key): key(key) {}
+ bool operator()(const Entry& entry) const {
+ return ExtractKey()(entry) < this->key;
+ }
+private:
+ Key key;
+};
+
+} /* namespace datasketches */
+
+#endif
diff --git a/tuple/include/theta_constants.hpp b/tuple/include/theta_constants.hpp
new file mode 100644
index 0000000..989681f
--- /dev/null
+++ b/tuple/include/theta_constants.hpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef THETA_CONSTANTS_HPP_
+#define THETA_CONSTANTS_HPP_
+
+namespace datasketches {
+
+namespace theta_constants {
+ enum resize_factor { X1, X2, X4, X8 };
+ static const uint64_t MAX_THETA = LLONG_MAX; // signed max for compatibility with Java
+ static const uint8_t MIN_LG_K = 5;
+ static const uint8_t MAX_LG_K = 26;
+}
+
+} /* namespace datasketches */
+
+#endif
diff --git a/tuple/include/theta_helpers.hpp b/tuple/include/theta_helpers.hpp
new file mode 100644
index 0000000..6852590
--- /dev/null
+++ b/tuple/include/theta_helpers.hpp
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef THETA_HELPERS_HPP_
+#define THETA_HELPERS_HPP_
+
+#include <string>
+#include <stdexcept>
+
+namespace datasketches {
+
+template<typename T>
+static void check_value(T actual, T expected, const char* description) {
+ if (actual != expected) {
+ throw std::invalid_argument(std::string(description) + " mismatch: expected " + std::to_string(expected) + ", actual " + std::to_string(actual));
+ }
+}
+
+template<bool dummy>
+class checker {
+public:
+ static void check_serial_version(uint8_t actual, uint8_t expected) {
+ check_value(actual, expected, "serial version");
+ }
+ static void check_sketch_family(uint8_t actual, uint8_t expected) {
+ check_value(actual, expected, "sketch family");
+ }
+ static void check_sketch_type(uint8_t actual, uint8_t expected) {
+ check_value(actual, expected, "sketch type");
+ }
+ static void check_seed_hash(uint16_t actual, uint16_t expected) {
+ check_value(actual, expected, "seed hash");
+ }
+};
+
+} /* namespace datasketches */
+
+#endif
diff --git a/tuple/include/theta_intersection_base.hpp b/tuple/include/theta_intersection_base.hpp
new file mode 100644
index 0000000..c034590
--- /dev/null
+++ b/tuple/include/theta_intersection_base.hpp
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef THETA_INTERSECTION_BASE_HPP_
+#define THETA_INTERSECTION_BASE_HPP_
+
+namespace datasketches {
+
+template<
+ typename Entry,
+ typename ExtractKey,
+ typename Policy,
+ typename Sketch,
+ typename CompactSketch,
+ typename Allocator
+>
+class theta_intersection_base {
+public:
+ using hash_table = theta_update_sketch_base<Entry, ExtractKey, Allocator>;
+ using resize_factor = typename hash_table::resize_factor;
+ using comparator = compare_by_key<ExtractKey>;
+ theta_intersection_base(uint64_t seed, const Policy& policy, const Allocator& allocator);
+
+ template<typename FwdSketch>
+ void update(FwdSketch&& sketch);
+
+ CompactSketch get_result(bool ordered = true) const;
+
+ bool has_result() const;
+
+ const Policy& get_policy() const;
+
+private:
+ Policy policy_;
+ bool is_valid_;
+ hash_table table_;
+};
+
+} /* namespace datasketches */
+
+#include "theta_intersection_base_impl.hpp"
+
+#endif
diff --git a/tuple/include/theta_intersection_base_impl.hpp b/tuple/include/theta_intersection_base_impl.hpp
new file mode 100644
index 0000000..286f0ca
--- /dev/null
+++ b/tuple/include/theta_intersection_base_impl.hpp
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <iostream>
+#include <sstream>
+#include <algorithm>
+
+#include "conditional_forward.hpp"
+
+namespace datasketches {
+
+template<typename EN, typename EK, typename P, typename S, typename CS, typename A>
+theta_intersection_base<EN, EK, P, S, CS, A>::theta_intersection_base(uint64_t seed, const P& policy, const A& allocator):
+policy_(policy),
+is_valid_(false),
+table_(0, 0, resize_factor::X1, theta_constants::MAX_THETA, seed, allocator, false)
+{}
+
+template<typename EN, typename EK, typename P, typename S, typename CS, typename A>
+template<typename SS>
+void theta_intersection_base<EN, EK, P, S, CS, A>::update(SS&& sketch) {
+ if (table_.is_empty_) return;
+ if (!sketch.is_empty() && sketch.get_seed_hash() != compute_seed_hash(table_.seed_)) throw std::invalid_argument("seed hash mismatch");
+ table_.is_empty_ |= sketch.is_empty();
+ table_.theta_ = std::min(table_.theta_, sketch.get_theta64());
+ if (is_valid_ && table_.num_entries_ == 0) return;
+ if (sketch.get_num_retained() == 0) {
+ is_valid_ = true;
+ table_ = hash_table(0, 0, resize_factor::X1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
+ return;
+ }
+ if (!is_valid_) { // first update, copy or move incoming sketch
+ is_valid_ = true;
+ const uint8_t lg_size = lg_size_from_count(sketch.get_num_retained(), theta_update_sketch_base<EN, EK, A>::REBUILD_THRESHOLD);
+ table_ = hash_table(lg_size, lg_size, resize_factor::X1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
+ for (auto& entry: sketch) {
+ auto result = table_.find(EK()(entry));
+ if (result.second) {
+ throw std::invalid_argument("duplicate key, possibly corrupted input sketch");
+ }
+ table_.insert(result.first, conditional_forward<SS>(entry));
+ }
+ if (table_.num_entries_ != sketch.get_num_retained()) throw std::invalid_argument("num entries mismatch, possibly corrupted input sketch");
+ } else { // intersection
+ const uint32_t max_matches = std::min(table_.num_entries_, sketch.get_num_retained());
+ std::vector<EN, A> matched_entries(table_.allocator_);
+ matched_entries.reserve(max_matches);
+ uint32_t match_count = 0;
+ uint32_t count = 0;
+ for (auto& entry: sketch) {
+ if (EK()(entry) < table_.theta_) {
+ auto result = table_.find(EK()(entry));
+ if (result.second) {
+ if (match_count == max_matches) throw std::invalid_argument("max matches exceeded, possibly corrupted input sketch");
+ policy_(*result.first, conditional_forward<SS>(entry));
+ matched_entries.push_back(std::move(*result.first));
+ ++match_count;
+ }
+ } else if (sketch.is_ordered()) {
+ break; // early stop
+ }
+ ++count;
+ }
+ if (count > sketch.get_num_retained()) {
+ throw std::invalid_argument(" more keys than expected, possibly corrupted input sketch");
+ } else if (!sketch.is_ordered() && count < sketch.get_num_retained()) {
+ throw std::invalid_argument(" fewer keys than expected, possibly corrupted input sketch");
+ }
+ if (match_count == 0) {
+ table_ = hash_table(0, 0, resize_factor::X1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
+ if (table_.theta_ == theta_constants::MAX_THETA) table_.is_empty_ = true;
+ } else {
+ const uint8_t lg_size = lg_size_from_count(match_count, theta_update_sketch_base<EN, EK, A>::REBUILD_THRESHOLD);
+ table_ = hash_table(lg_size, lg_size, resize_factor::X1, table_.theta_, table_.seed_, table_.allocator_, table_.is_empty_);
+ for (uint32_t i = 0; i < match_count; i++) {
+ auto result = table_.find(EK()(matched_entries[i]));
+ table_.insert(result.first, std::move(matched_entries[i]));
+ }
+ }
+ }
+}
+
+template<typename EN, typename EK, typename P, typename S, typename CS, typename A>
+CS theta_intersection_base<EN, EK, P, S, CS, A>::get_result(bool ordered) const {
+ if (!is_valid_) throw std::invalid_argument("calling get_result() before calling update() is undefined");
+ std::vector<EN, A> entries(table_.allocator_);
+ if (table_.num_entries_ > 0) {
+ entries.reserve(table_.num_entries_);
+ std::copy_if(table_.begin(), table_.end(), std::back_inserter(entries), key_not_zero<EN, EK>());
+ if (ordered) std::sort(entries.begin(), entries.end(), comparator());
+ }
+ return CS(table_.is_empty_, ordered, compute_seed_hash(table_.seed_), table_.theta_, std::move(entries));
+}
+
+template<typename EN, typename EK, typename P, typename S, typename CS, typename A>
+bool theta_intersection_base<EN, EK, P, S, CS, A>::has_result() const {
+ return is_valid_;
+}
+
+template<typename EN, typename EK, typename P, typename S, typename CS, typename A>
+const P& theta_intersection_base<EN, EK, P, S, CS, A>::get_policy() const {
+ return policy_;
+}
+
+} /* namespace datasketches */
diff --git a/tuple/include/theta_intersection_experimental.hpp b/tuple/include/theta_intersection_experimental.hpp
new file mode 100644
index 0000000..293b2e9
--- /dev/null
+++ b/tuple/include/theta_intersection_experimental.hpp
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef THETA_INTERSECTION_EXPERIMENTAL_HPP_
+#define THETA_INTERSECTION_EXPERIMENTAL_HPP_
+
+#include "theta_sketch_experimental.hpp"
+#include "theta_intersection_base.hpp"
+
+namespace datasketches {
+
+template<typename Allocator = std::allocator<uint64_t>>
+class theta_intersection_experimental {
+public:
+ using Entry = uint64_t;
+ using ExtractKey = trivial_extract_key;
+ using Sketch = theta_sketch_experimental<Allocator>;
+ using CompactSketch = compact_theta_sketch_experimental<Allocator>;
+
+ struct pass_through_policy {
+ uint64_t operator()(uint64_t internal_entry, uint64_t incoming_entry) const {
+ unused(incoming_entry);
+ return internal_entry;
+ }
+ };
+ using State = theta_intersection_base<Entry, ExtractKey, pass_through_policy, Sketch, CompactSketch, Allocator>;
+
+ explicit theta_intersection_experimental(uint64_t seed = DEFAULT_SEED, const Allocator& allocator = Allocator());
+
+ /**
+ * Updates the intersection with a given sketch.
+ * The intersection can be viewed as starting from the "universe" set, and every update
+ * can reduce the current set to leave the overlapping subset only.
+ * @param sketch represents input set for the intersection
+ */
+ template<typename FwdSketch>
+ void update(FwdSketch&& sketch);
+
+ /**
+ * Produces a copy of the current state of the intersection.
+ * If update() was not called, the state is the infinite "universe",
+ * which is considered an undefined state, and throws an exception.
+ * @param ordered optional flag to specify if ordered sketch should be produced
+ * @return the result of the intersection
+ */
+ CompactSketch get_result(bool ordered = true) const;
+
+ /**
+ * Returns true if the state of the intersection is defined (not infinite "universe").
+ * @return true if the state is valid
+ */
+ bool has_result() const;
+
+private:
+ State state_;
+};
+
+} /* namespace datasketches */
+
+#include "theta_intersection_experimental_impl.hpp"
+
+#endif
diff --git a/tuple/include/theta_intersection_experimental_impl.hpp b/tuple/include/theta_intersection_experimental_impl.hpp
new file mode 100644
index 0000000..e8bcfbb
--- /dev/null
+++ b/tuple/include/theta_intersection_experimental_impl.hpp
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+namespace datasketches {
+
+template<typename A>
+theta_intersection_experimental<A>::theta_intersection_experimental(uint64_t seed, const A& allocator):
+state_(seed, pass_through_policy(), allocator)
+{}
+
+template<typename A>
+template<typename SS>
+void theta_intersection_experimental<A>::update(SS&& sketch) {
+ state_.update(std::forward<SS>(sketch));
+}
+
+template<typename A>
+auto theta_intersection_experimental<A>::get_result(bool ordered) const -> CompactSketch {
+ return state_.get_result(ordered);
+}
+
+template<typename A>
+bool theta_intersection_experimental<A>::has_result() const {
+ return state_.has_result();
+}
+
+} /* namespace datasketches */
diff --git a/tuple/include/theta_set_difference_base.hpp b/tuple/include/theta_set_difference_base.hpp
new file mode 100644
index 0000000..5cc601f
--- /dev/null
+++ b/tuple/include/theta_set_difference_base.hpp
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef THETA_SET_DIFFERENCE_BASE_HPP_
+#define THETA_SET_DIFFERENCE_BASE_HPP_
+
+#include "theta_comparators.hpp"
+#include "theta_update_sketch_base.hpp"
+
+namespace datasketches {
+
+template<
+ typename Entry,
+ typename ExtractKey,
+ typename CompactSketch,
+ typename Allocator
+>
+class theta_set_difference_base {
+public:
+ using comparator = compare_by_key<ExtractKey>;
+ using AllocU64 = typename std::allocator_traits<Allocator>::template rebind_alloc<uint64_t>;
+ using hash_table = theta_update_sketch_base<uint64_t, trivial_extract_key, AllocU64>;
+
+ theta_set_difference_base(uint64_t seed, const Allocator& allocator = Allocator());
+
+ template<typename FwdSketch, typename Sketch>
+ CompactSketch compute(FwdSketch&& a, const Sketch& b, bool ordered) const;
+
+private:
+ Allocator allocator_;
+ uint16_t seed_hash_;
+};
+
+} /* namespace datasketches */
+
+#include "theta_set_difference_base_impl.hpp"
+
+#endif
diff --git a/tuple/include/theta_set_difference_base_impl.hpp b/tuple/include/theta_set_difference_base_impl.hpp
new file mode 100644
index 0000000..14708f9
--- /dev/null
+++ b/tuple/include/theta_set_difference_base_impl.hpp
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <algorithm>
+
+#include "conditional_back_inserter.hpp"
+#include "conditional_forward.hpp"
+
+namespace datasketches {
+
+template<typename EN, typename EK, typename CS, typename A>
+theta_set_difference_base<EN, EK, CS, A>::theta_set_difference_base(uint64_t seed, const A& allocator):
+allocator_(allocator),
+seed_hash_(compute_seed_hash(seed))
+{}
+
+template<typename EN, typename EK, typename CS, typename A>
+template<typename FwdSketch, typename Sketch>
+CS theta_set_difference_base<EN, EK, CS, A>::compute(FwdSketch&& a, const Sketch& b, bool ordered) const {
+ if (a.is_empty() || a.get_num_retained() == 0 || b.is_empty()) return CS(a, ordered);
+ if (a.get_seed_hash() != seed_hash_) throw std::invalid_argument("A seed hash mismatch");
+ if (b.get_seed_hash() != seed_hash_) throw std::invalid_argument("B seed hash mismatch");
+
+ const uint64_t theta = std::min(a.get_theta64(), b.get_theta64());
+ std::vector<EN, A> entries(allocator_);
+ bool is_empty = a.is_empty();
+
+ if (b.get_num_retained() == 0) {
+ std::copy_if(forward_begin(std::forward<FwdSketch>(a)), forward_end(std::forward<FwdSketch>(a)), std::back_inserter(entries),
+ key_less_than<uint64_t, EN, EK>(theta));
+ } else {
+ if (a.is_ordered() && b.is_ordered()) { // sort-based
+ std::set_difference(forward_begin(std::forward<FwdSketch>(a)), forward_end(std::forward<FwdSketch>(a)), b.begin(), b.end(),
+ conditional_back_inserter(entries, key_less_than<uint64_t, EN, EK>(theta)), comparator());
+ } else { // hash-based
+ const uint8_t lg_size = lg_size_from_count(b.get_num_retained(), hash_table::REBUILD_THRESHOLD);
+ hash_table table(lg_size, lg_size, hash_table::resize_factor::X1, 0, 0, allocator_); // theta and seed are not used here
+ for (const auto& entry: b) {
+ const uint64_t hash = EK()(entry);
+ if (hash < theta) {
+ table.insert(table.find(hash).first, hash);
+ } else if (b.is_ordered()) {
+ break; // early stop
+ }
+ }
+
+ // scan A lookup B
+ for (auto& entry: a) {
+ const uint64_t hash = EK()(entry);
+ if (hash < theta) {
+ auto result = table.find(hash);
+ if (!result.second) entries.push_back(conditional_forward<FwdSketch>(entry));
+ } else if (a.is_ordered()) {
+ break; // early stop
+ }
+ }
+ }
+ }
+ if (entries.empty() && theta == theta_constants::MAX_THETA) is_empty = true;
+ if (ordered && !a.is_ordered()) std::sort(entries.begin(), entries.end(), comparator());
+ return CS(is_empty, a.is_ordered() || ordered, seed_hash_, theta, std::move(entries));
+}
+
+} /* namespace datasketches */
diff --git a/tuple/include/theta_sketch_experimental.hpp b/tuple/include/theta_sketch_experimental.hpp
new file mode 100644
index 0000000..2056687
--- /dev/null
+++ b/tuple/include/theta_sketch_experimental.hpp
@@ -0,0 +1,393 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef THETA_SKETCH_EXPERIMENTAL_HPP_
+#define THETA_SKETCH_EXPERIMENTAL_HPP_
+
+#include "theta_update_sketch_base.hpp"
+
+namespace datasketches {
+
+// experimental theta sketch derived from the same base as tuple sketch
+
+template<typename Allocator = std::allocator<uint64_t>>
+class theta_sketch_experimental {
+public:
+ using Entry = uint64_t;
+ using ExtractKey = trivial_extract_key;
+ using iterator = theta_iterator<Entry, ExtractKey>;
+ using const_iterator = theta_const_iterator<Entry, ExtractKey>;
+
+ virtual ~theta_sketch_experimental() = default;
+
+ /**
+ * @return allocator
+ */
+ virtual Allocator get_allocator() const = 0;
+
+ /**
+ * @return true if this sketch represents an empty set (not the same as no retained entries!)
+ */
+ virtual bool is_empty() const = 0;
+
+ /**
+ * @return estimate of the distinct count of the input stream
+ */
+ double get_estimate() const;
+
+ /**
+ * Returns the approximate lower error bound given a number of standard deviations.
+ * This parameter is similar to the number of standard deviations of the normal distribution
+ * and corresponds to approximately 67%, 95% and 99% confidence intervals.
+ * @param num_std_devs number of Standard Deviations (1, 2 or 3)
+ * @return the lower bound
+ */
+ double get_lower_bound(uint8_t num_std_devs) const;
+
+ /**
+ * Returns the approximate upper error bound given a number of standard deviations.
+ * This parameter is similar to the number of standard deviations of the normal distribution
+ * and corresponds to approximately 67%, 95% and 99% confidence intervals.
+ * @param num_std_devs number of Standard Deviations (1, 2 or 3)
+ * @return the upper bound
+ */
+ double get_upper_bound(uint8_t num_std_devs) const;
+
+ /**
+ * @return true if the sketch is in estimation mode (as opposed to exact mode)
+ */
+ bool is_estimation_mode() const;
+
+ /**
+ * @return theta as a fraction from 0 to 1 (effective sampling rate)
+ */
+ double get_theta() const;
+
+ /**
+ * @return theta as a positive integer between 0 and LLONG_MAX
+ */
+ virtual uint64_t get_theta64() const = 0;
+
+ /**
+ * @return the number of retained entries in the sketch
+ */
+ virtual uint32_t get_num_retained() const = 0;
+
+ /**
+ * @return hash of the seed that was used to hash the input
+ */
+ virtual uint16_t get_seed_hash() const = 0;
+
+ /**
+ * @return true if retained entries are ordered
+ */
+ virtual bool is_ordered() const = 0;
+
+ /**
+ * Provides a human-readable summary of this sketch as a string
+ * @param print_items if true include the list of items retained by the sketch
+ * @return sketch summary as a string
+ */
+ virtual string<Allocator> to_string(bool print_items = false) const;
+
+ /**
+ * Iterator over hash values in this sketch.
+ * @return begin iterator
+ */
+ virtual iterator begin() = 0;
+
+ /**
+ * Iterator pointing past the valid range.
+ * Not to be incremented or dereferenced.
+ * @return end iterator
+ */
+ virtual iterator end() = 0;
+
+ /**
+ * Const iterator over hash values in this sketch.
+ * @return begin iterator
+ */
+ virtual const_iterator begin() const = 0;
+
+ /**
+ * Const iterator pointing past the valid range.
+ * Not to be incremented or dereferenced.
+ * @return end iterator
+ */
+ virtual const_iterator end() const = 0;
+
+protected:
+ virtual void print_specifics(std::ostringstream& os) const = 0;
+};
+
+// forward declaration
+template<typename A> class compact_theta_sketch_experimental;
+
+template<typename Allocator = std::allocator<uint64_t>>
+class update_theta_sketch_experimental: public theta_sketch_experimental<Allocator> {
+public:
+ using Base = theta_sketch_experimental<Allocator>;
+ using Entry = typename Base::Entry;
+ using ExtractKey = typename Base::ExtractKey;
+ using iterator = typename Base::iterator;
+ using const_iterator = typename Base::const_iterator;
+ using theta_table = theta_update_sketch_base<Entry, ExtractKey, Allocator>;
+ using resize_factor = typename theta_table::resize_factor;
+
+ // No constructor here. Use builder instead.
+ class builder;
+
+ update_theta_sketch_experimental(const update_theta_sketch_experimental&) = default;
+ update_theta_sketch_experimental(update_theta_sketch_experimental&&) noexcept = default;
+ virtual ~update_theta_sketch_experimental() = default;
+ update_theta_sketch_experimental& operator=(const update_theta_sketch_experimental&) = default;
+ update_theta_sketch_experimental& operator=(update_theta_sketch_experimental&&) = default;
+
+ virtual Allocator get_allocator() const;
+ virtual bool is_empty() const;
+ virtual bool is_ordered() const;
+ virtual uint16_t get_seed_hash() const;
+ virtual uint64_t get_theta64() const;
+ virtual uint32_t get_num_retained() const;
+
+ /**
+ * @return configured nominal number of entries in the sketch
+ */
+ uint8_t get_lg_k() const;
+
+ /**
+ * @return configured resize factor of the sketch
+ */
+ resize_factor get_rf() const;
+
+ /**
+ * Update this sketch with a given string.
+ * @param value string to update the sketch with
+ */
+ void update(const std::string& value);
+
+ /**
+ * Update this sketch with a given unsigned 64-bit integer.
+ * @param value uint64_t to update the sketch with
+ */
+ void update(uint64_t value);
+
+ /**
+ * Update this sketch with a given signed 64-bit integer.
+ * @param value int64_t to update the sketch with
+ */
+ void update(int64_t value);
+
+ /**
+ * Update this sketch with a given unsigned 32-bit integer.
+ * For compatibility with Java implementation.
+ * @param value uint32_t to update the sketch with
+ */
+ void update(uint32_t value);
+
+ /**
+ * Update this sketch with a given signed 32-bit integer.
+ * For compatibility with Java implementation.
+ * @param value int32_t to update the sketch with
+ */
+ void update(int32_t value);
+
+ /**
+ * Update this sketch with a given unsigned 16-bit integer.
+ * For compatibility with Java implementation.
+ * @param value uint16_t to update the sketch with
+ */
+ void update(uint16_t value);
+
+ /**
+ * Update this sketch with a given signed 16-bit integer.
+ * For compatibility with Java implementation.
+ * @param value int16_t to update the sketch with
+ */
+ void update(int16_t value);
+
+ /**
+ * Update this sketch with a given unsigned 8-bit integer.
+ * For compatibility with Java implementation.
+ * @param value uint8_t to update the sketch with
+ */
+ void update(uint8_t value);
+
+ /**
+ * Update this sketch with a given signed 8-bit integer.
+ * For compatibility with Java implementation.
+ * @param value int8_t to update the sketch with
+ */
+ void update(int8_t value);
+
+ /**
+ * Update this sketch with a given double-precision floating point value.
+ * For compatibility with Java implementation.
+ * @param value double to update the sketch with
+ */
+ void update(double value);
+
+ /**
+ * Update this sketch with a given floating point value.
+ * For compatibility with Java implementation.
+ * @param value float to update the sketch with
+ */
+ void update(float value);
+
+ /**
+ * Update this sketch with given data of any type.
+ * This is a "universal" update that covers all cases above,
+ * but may produce different hashes.
+ * Be very careful to hash input values consistently using the same approach
+ * both over time and on different platforms
+ * and while passing sketches between C++ environment and Java environment.
+ * Otherwise two sketches that should represent overlapping sets will be disjoint
+ * For instance, for signed 32-bit values call update(int32_t) method above,
+ * which does widening conversion to int64_t, if compatibility with Java is expected
+ * @param data pointer to the data
+ * @param length of the data in bytes
+ */
+ void update(const void* data, size_t length);
+
+ /**
+ * Remove retained entries in excess of the nominal size k (if any)
+ */
+ void trim();
+
+ /**
+ * Converts this sketch to a compact sketch (ordered or unordered).
+ * @param ordered optional flag to specify if ordered sketch should be produced
+ * @return compact sketch
+ */
+ compact_theta_sketch_experimental<Allocator> compact(bool ordered = true) const;
+
+ virtual iterator begin();
+ virtual iterator end();
+ virtual const_iterator begin() const;
+ virtual const_iterator end() const;
+
+private:
+ theta_table table_;
+
+ // for builder
+ update_theta_sketch_experimental(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta,
+ uint64_t seed, const Allocator& allocator);
+
+ virtual void print_specifics(std::ostringstream& os) const;
+};
+
+// compact sketch
+
+template<typename Allocator = std::allocator<uint64_t>>
+class compact_theta_sketch_experimental: public theta_sketch_experimental<Allocator> {
+public:
+ using Base = theta_sketch_experimental<Allocator>;
+ using iterator = typename Base::iterator;
+ using const_iterator = typename Base::const_iterator;
+ using AllocBytes = typename std::allocator_traits<Allocator>::template rebind_alloc<uint8_t>;
+ using vector_bytes = std::vector<uint8_t, AllocBytes>;
+
+ static const uint8_t SERIAL_VERSION = 3;
+ static const uint8_t SKETCH_TYPE = 3;
+
+ // Instances of this type can be obtained:
+ // - by compacting an update_theta_sketch
+ // - as a result of a set operation
+ // - by deserializing a previously serialized compact sketch
+
+ compact_theta_sketch_experimental(const Base& other, bool ordered);
+ compact_theta_sketch_experimental(const compact_theta_sketch_experimental&) = default;
+ compact_theta_sketch_experimental(compact_theta_sketch_experimental&&) noexcept = default;
+ virtual ~compact_theta_sketch_experimental() = default;
+ compact_theta_sketch_experimental& operator=(const compact_theta_sketch_experimental&) = default;
+ compact_theta_sketch_experimental& operator=(compact_theta_sketch_experimental&&) = default;
+
+ virtual Allocator get_allocator() const;
+ virtual bool is_empty() const;
+ virtual bool is_ordered() const;
+ virtual uint64_t get_theta64() const;
+ virtual uint32_t get_num_retained() const;
+ virtual uint16_t get_seed_hash() const;
+
+ /**
+ * This method serializes the sketch into a given stream in a binary form
+ * @param os output stream
+ */
+ void serialize(std::ostream& os) const;
+
+ /**
+ * This method serializes the sketch as a vector of bytes.
+ * An optional header can be reserved in front of the sketch.
+ * It is an uninitialized space of a given size.
+ * This header is used in Datasketches PostgreSQL extension.
+ * @param header_size_bytes space to reserve in front of the sketch
+ */
+ vector_bytes serialize(unsigned header_size_bytes = 0) const;
+
+ virtual iterator begin();
+ virtual iterator end();
+ virtual const_iterator begin() const;
+ virtual const_iterator end() const;
+
+ /**
+ * This method deserializes a sketch from a given stream.
+ * @param is input stream
+ * @param seed the seed for the hash function that was used to create the sketch
+ * @return an instance of the sketch
+ */
+ static compact_theta_sketch_experimental deserialize(std::istream& is,
+ uint64_t seed = DEFAULT_SEED, const Allocator& allocator = Allocator());
+
+ /**
+ * This method deserializes a sketch from a given array of bytes.
+ * @param bytes pointer to the array of bytes
+ * @param size the size of the array
+ * @param seed the seed for the hash function that was used to create the sketch
+ * @return an instance of the sketch
+ */
+ static compact_theta_sketch_experimental deserialize(const void* bytes, size_t size,
+ uint64_t seed = DEFAULT_SEED, const Allocator& allocator = Allocator());
+
+ // for internal use
+ compact_theta_sketch_experimental(bool is_empty, bool is_ordered, uint16_t seed_hash, uint64_t theta, std::vector<uint64_t, Allocator>&& entries);
+
+private:
+ enum flags { IS_BIG_ENDIAN, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED };
+
+ bool is_empty_;
+ bool is_ordered_;
+ uint16_t seed_hash_;
+ uint64_t theta_;
+ std::vector<uint64_t, Allocator> entries_;
+
+ virtual void print_specifics(std::ostringstream& os) const;
+};
+
+template<typename Allocator>
+class update_theta_sketch_experimental<Allocator>::builder: public theta_base_builder<builder, Allocator> {
+public:
+ builder(const Allocator& allocator = Allocator());
+ update_theta_sketch_experimental build() const;
+};
+
+} /* namespace datasketches */
+
+#include "theta_sketch_experimental_impl.hpp"
+
+#endif
diff --git a/tuple/include/theta_sketch_experimental_impl.hpp b/tuple/include/theta_sketch_experimental_impl.hpp
new file mode 100644
index 0000000..1fa4652
--- /dev/null
+++ b/tuple/include/theta_sketch_experimental_impl.hpp
@@ -0,0 +1,481 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <sstream>
+
+#include "serde.hpp"
+#include "binomial_bounds.hpp"
+#include "theta_helpers.hpp"
+
+namespace datasketches {
+
+template<typename A>
+bool theta_sketch_experimental<A>::is_estimation_mode() const {
+ return get_theta64() < theta_constants::MAX_THETA && !is_empty();
+}
+
+template<typename A>
+double theta_sketch_experimental<A>::get_theta() const {
+ return static_cast<double>(get_theta64()) / theta_constants::MAX_THETA;
+}
+
+template<typename A>
+double theta_sketch_experimental<A>::get_estimate() const {
+ return get_num_retained() / get_theta();
+}
+
+template<typename A>
+double theta_sketch_experimental<A>::get_lower_bound(uint8_t num_std_devs) const {
+ if (!is_estimation_mode()) return get_num_retained();
+ return binomial_bounds::get_lower_bound(get_num_retained(), get_theta(), num_std_devs);
+}
+
+template<typename A>
+double theta_sketch_experimental<A>::get_upper_bound(uint8_t num_std_devs) const {
+ if (!is_estimation_mode()) return get_num_retained();
+ return binomial_bounds::get_upper_bound(get_num_retained(), get_theta(), num_std_devs);
+}
+
+template<typename A>
+string<A> theta_sketch_experimental<A>::to_string(bool detail) const {
+ std::basic_ostringstream<char, std::char_traits<char>, AllocChar<A>> os;
+ os << "### Theta sketch summary:" << std::endl;
+ os << " num retained entries : " << get_num_retained() << std::endl;
+ os << " seed hash : " << get_seed_hash() << std::endl;
+ os << " empty? : " << (is_empty() ? "true" : "false") << std::endl;
+ os << " ordered? : " << (is_ordered() ? "true" : "false") << std::endl;
+ os << " estimation mode? : " << (is_estimation_mode() ? "true" : "false") << std::endl;
+ os << " theta (fraction) : " << get_theta() << std::endl;
+ os << " theta (raw 64-bit) : " << get_theta64() << std::endl;
+ os << " estimate : " << this->get_estimate() << std::endl;
+ os << " lower bound 95% conf : " << this->get_lower_bound(2) << std::endl;
+ os << " upper bound 95% conf : " << this->get_upper_bound(2) << std::endl;
+ print_specifics(os);
+ os << "### End sketch summary" << std::endl;
+ if (detail) {
+ os << "### Retained entries" << std::endl;
+ for (const auto& hash: *this) {
+ os << hash << std::endl;
+ }
+ os << "### End retained entries" << std::endl;
+ }
+ return os.str();
+}
+
+// update sketch
+
+template<typename A>
+update_theta_sketch_experimental<A>::update_theta_sketch_experimental(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf,
+ uint64_t theta, uint64_t seed, const A& allocator):
+table_(lg_cur_size, lg_nom_size, rf, theta, seed, allocator)
+{}
+
+template<typename A>
+A update_theta_sketch_experimental<A>::get_allocator() const {
+ return table_.allocator_;
+}
+
+template<typename A>
+bool update_theta_sketch_experimental<A>::is_empty() const {
+ return table_.is_empty_;
+}
+
+template<typename A>
+bool update_theta_sketch_experimental<A>::is_ordered() const {
+ return false;
+}
+
+template<typename A>
+uint64_t update_theta_sketch_experimental<A>::get_theta64() const {
+ return table_.theta_;
+}
+
+template<typename A>
+uint32_t update_theta_sketch_experimental<A>::get_num_retained() const {
+ return table_.num_entries_;
+}
+
+template<typename A>
+uint16_t update_theta_sketch_experimental<A>::get_seed_hash() const {
+ return compute_seed_hash(table_.seed_);
+}
+
+template<typename A>
+uint8_t update_theta_sketch_experimental<A>::get_lg_k() const {
+ return table_.lg_nom_size_;
+}
+
+template<typename A>
+auto update_theta_sketch_experimental<A>::get_rf() const -> resize_factor {
+ return table_.rf_;
+}
+
+template<typename A>
+void update_theta_sketch_experimental<A>::update(uint64_t value) {
+ update(&value, sizeof(value));
+}
+
+template<typename A>
+void update_theta_sketch_experimental<A>::update(int64_t value) {
+ update(&value, sizeof(value));
+}
+
+template<typename A>
+void update_theta_sketch_experimental<A>::update(uint32_t value) {
+ update(static_cast<int32_t>(value));
+}
+
+template<typename A>
+void update_theta_sketch_experimental<A>::update(int32_t value) {
+ update(static_cast<int64_t>(value));
+}
+
+template<typename A>
+void update_theta_sketch_experimental<A>::update(uint16_t value) {
+ update(static_cast<int16_t>(value));
+}
+
+template<typename A>
+void update_theta_sketch_experimental<A>::update(int16_t value) {
+ update(static_cast<int64_t>(value));
+}
+
+template<typename A>
+void update_theta_sketch_experimental<A>::update(uint8_t value) {
+ update(static_cast<int8_t>(value));
+}
+
+template<typename A>
+void update_theta_sketch_experimental<A>::update(int8_t value) {
+ update(static_cast<int64_t>(value));
+}
+
+template<typename A>
+void update_theta_sketch_experimental<A>::update(double value) {
+ update(canonical_double(value));
+}
+
+template<typename A>
+void update_theta_sketch_experimental<A>::update(float value) {
+ update(static_cast<double>(value));
+}
+
+template<typename A>
+void update_theta_sketch_experimental<A>::update(const std::string& value) {
+ if (value.empty()) return;
+ update(value.c_str(), value.length());
+}
+
+template<typename A>
+void update_theta_sketch_experimental<A>::update(const void* data, size_t length) {
+ const uint64_t hash = table_.hash_and_screen(data, length);
+ if (hash == 0) return;
+ auto result = table_.find(hash);
+ if (!result.second) {
+ table_.insert(result.first, hash);
+ }
+}
+
+template<typename A>
+void update_theta_sketch_experimental<A>::trim() {
+ table_.trim();
+}
+
+template<typename A>
+auto update_theta_sketch_experimental<A>::begin() -> iterator {
+ return iterator(table_.entries_, 1 << table_.lg_cur_size_, 0);
+}
+
+template<typename A>
+auto update_theta_sketch_experimental<A>::end() -> iterator {
+ return iterator(nullptr, 0, 1 << table_.lg_cur_size_);
+}
+
+template<typename A>
+auto update_theta_sketch_experimental<A>::begin() const -> const_iterator {
+ return const_iterator(table_.entries_, 1 << table_.lg_cur_size_, 0);
+}
+
+template<typename A>
+auto update_theta_sketch_experimental<A>::end() const -> const_iterator {
+ return const_iterator(nullptr, 0, 1 << table_.lg_cur_size_);
+}
+template<typename A>
+compact_theta_sketch_experimental<A> update_theta_sketch_experimental<A>::compact(bool ordered) const {
+ return compact_theta_sketch_experimental<A>(*this, ordered);
+}
+
+template<typename A>
+void update_theta_sketch_experimental<A>::print_specifics(std::ostringstream& os) const {
+ os << " lg nominal size : " << static_cast<int>(table_.lg_nom_size_) << std::endl;
+ os << " lg current size : " << static_cast<int>(table_.lg_cur_size_) << std::endl;
+ os << " resize factor : " << (1 << table_.rf_) << std::endl;
+}
+
+// builder
+
+template<typename A>
+update_theta_sketch_experimental<A>::builder::builder(const A& allocator): theta_base_builder<builder, A>(allocator) {}
+
+template<typename A>
+update_theta_sketch_experimental<A> update_theta_sketch_experimental<A>::builder::build() const {
+ return update_theta_sketch_experimental(this->starting_lg_size(), this->lg_k_, this->rf_, this->starting_theta(), this->seed_, this->allocator_);
+}
+
+// experimental compact theta sketch
+
+template<typename A>
+compact_theta_sketch_experimental<A>::compact_theta_sketch_experimental(const Base& other, bool ordered):
+is_empty_(other.is_empty()),
+is_ordered_(other.is_ordered() || ordered),
+seed_hash_(other.get_seed_hash()),
+theta_(other.get_theta64()),
+entries_(other.get_allocator())
+{
+ entries_.reserve(other.get_num_retained());
+ std::copy(other.begin(), other.end(), std::back_inserter(entries_));
+ if (ordered && !other.is_ordered()) std::sort(entries_.begin(), entries_.end());
+}
+
+template<typename A>
+compact_theta_sketch_experimental<A>::compact_theta_sketch_experimental(bool is_empty, bool is_ordered, uint16_t seed_hash, uint64_t theta,
+ std::vector<uint64_t, A>&& entries):
+is_empty_(is_empty),
+is_ordered_(is_ordered),
+seed_hash_(seed_hash),
+theta_(theta),
+entries_(std::move(entries))
+{}
+
+template<typename A>
+A compact_theta_sketch_experimental<A>::get_allocator() const {
+ return entries_.get_allocator();
+}
+
+template<typename A>
+bool compact_theta_sketch_experimental<A>::is_empty() const {
+ return is_empty_;
+}
+
+template<typename A>
+bool compact_theta_sketch_experimental<A>::is_ordered() const {
+ return is_ordered_;
+}
+
+template<typename A>
+uint64_t compact_theta_sketch_experimental<A>::get_theta64() const {
+ return theta_;
+}
+
+template<typename A>
+uint32_t compact_theta_sketch_experimental<A>::get_num_retained() const {
+ return entries_.size();
+}
+
+template<typename A>
+uint16_t compact_theta_sketch_experimental<A>::get_seed_hash() const {
+ return seed_hash_;
+}
+
+template<typename A>
+auto compact_theta_sketch_experimental<A>::begin() -> iterator {
+ return iterator(entries_.data(), entries_.size(), 0);
+}
+
+template<typename A>
+auto compact_theta_sketch_experimental<A>::end() -> iterator {
+ return iterator(nullptr, 0, entries_.size());
+}
+
+template<typename A>
+auto compact_theta_sketch_experimental<A>::begin() const -> const_iterator {
+ return const_iterator(entries_.data(), entries_.size(), 0);
+}
+
+template<typename A>
+auto compact_theta_sketch_experimental<A>::end() const -> const_iterator {
+ return const_iterator(nullptr, 0, entries_.size());
+}
+
+template<typename A>
+void compact_theta_sketch_experimental<A>::print_specifics(std::ostringstream&) const {}
+
+template<typename A>
+void compact_theta_sketch_experimental<A>::serialize(std::ostream& os) const {
+ const bool is_single_item = entries_.size() == 1 && !this->is_estimation_mode();
+ const uint8_t preamble_longs = this->is_empty() || is_single_item ? 1 : this->is_estimation_mode() ? 3 : 2;
+ os.write(reinterpret_cast<const char*>(&preamble_longs), sizeof(preamble_longs));
+ const uint8_t serial_version = SERIAL_VERSION;
+ os.write(reinterpret_cast<const char*>(&serial_version), sizeof(serial_version));
+ const uint8_t type = SKETCH_TYPE;
+ os.write(reinterpret_cast<const char*>(&type), sizeof(type));
+ const uint16_t unused16 = 0;
+ os.write(reinterpret_cast<const char*>(&unused16), sizeof(unused16));
+ const uint8_t flags_byte(
+ (1 << flags::IS_COMPACT) |
+ (1 << flags::IS_READ_ONLY) |
+ (this->is_empty() ? 1 << flags::IS_EMPTY : 0) |
+ (this->is_ordered() ? 1 << flags::IS_ORDERED : 0)
+ );
+ os.write(reinterpret_cast<const char*>(&flags_byte), sizeof(flags_byte));
+ const uint16_t seed_hash = get_seed_hash();
+ os.write(reinterpret_cast<const char*>(&seed_hash), sizeof(seed_hash));
+ if (!this->is_empty()) {
+ if (!is_single_item) {
+ const uint32_t num_entries = entries_.size();
+ os.write(reinterpret_cast<const char*>(&num_entries), sizeof(num_entries));
+ const uint32_t unused32 = 0;
+ os.write(reinterpret_cast<const char*>(&unused32), sizeof(unused32));
+ if (this->is_estimation_mode()) {
+ os.write(reinterpret_cast<const char*>(&(this->theta_)), sizeof(uint64_t));
+ }
+ }
+ os.write(reinterpret_cast<const char*>(entries_.data()), entries_.size() * sizeof(uint64_t));
+ }
+}
+
+template<typename A>
+auto compact_theta_sketch_experimental<A>::serialize(unsigned header_size_bytes) const -> vector_bytes {
+ const bool is_single_item = entries_.size() == 1 && !this->is_estimation_mode();
+ const uint8_t preamble_longs = this->is_empty() || is_single_item ? 1 : this->is_estimation_mode() ? 3 : 2;
+ const size_t size = header_size_bytes + sizeof(uint64_t) * preamble_longs
+ + sizeof(uint64_t) * entries_.size();
+ vector_bytes bytes(size, 0, entries_.get_allocator());
+ uint8_t* ptr = bytes.data() + header_size_bytes;
+
+ ptr += copy_to_mem(&preamble_longs, ptr, sizeof(preamble_longs));
+ const uint8_t serial_version = SERIAL_VERSION;
+ ptr += copy_to_mem(&serial_version, ptr, sizeof(serial_version));
+ const uint8_t type = SKETCH_TYPE;
+ ptr += copy_to_mem(&type, ptr, sizeof(type));
+ const uint16_t unused16 = 0;
+ ptr += copy_to_mem(&unused16, ptr, sizeof(unused16));
+ const uint8_t flags_byte(
+ (1 << flags::IS_COMPACT) |
+ (1 << flags::IS_READ_ONLY) |
+ (this->is_empty() ? 1 << flags::IS_EMPTY : 0) |
+ (this->is_ordered() ? 1 << flags::IS_ORDERED : 0)
+ );
+ ptr += copy_to_mem(&flags_byte, ptr, sizeof(flags_byte));
+ const uint16_t seed_hash = get_seed_hash();
+ ptr += copy_to_mem(&seed_hash, ptr, sizeof(seed_hash));
+ if (!this->is_empty()) {
+ if (!is_single_item) {
+ const uint32_t num_entries = entries_.size();
+ ptr += copy_to_mem(&num_entries, ptr, sizeof(num_entries));
+ const uint32_t unused32 = 0;
+ ptr += copy_to_mem(&unused32, ptr, sizeof(unused32));
+ if (this->is_estimation_mode()) {
+ ptr += copy_to_mem(&theta_, ptr, sizeof(uint64_t));
+ }
+ }
+ ptr += copy_to_mem(entries_.data(), ptr, entries_.size() * sizeof(uint64_t));
+ }
+ return bytes;
+}
+
+template<typename A>
+compact_theta_sketch_experimental<A> compact_theta_sketch_experimental<A>::deserialize(std::istream& is, uint64_t seed, const A& allocator) {
+ uint8_t preamble_longs;
+ is.read(reinterpret_cast<char*>(&preamble_longs), sizeof(preamble_longs));
+ uint8_t serial_version;
+ is.read(reinterpret_cast<char*>(&serial_version), sizeof(serial_version));
+ uint8_t type;
+ is.read(reinterpret_cast<char*>(&type), sizeof(type));
+ uint16_t unused16;
+ is.read(reinterpret_cast<char*>(&unused16), sizeof(unused16));
+ uint8_t flags_byte;
+ is.read(reinterpret_cast<char*>(&flags_byte), sizeof(flags_byte));
+ uint16_t seed_hash;
+ is.read(reinterpret_cast<char*>(&seed_hash), sizeof(seed_hash));
+ checker<true>::check_sketch_type(type, SKETCH_TYPE);
+ checker<true>::check_serial_version(serial_version, SERIAL_VERSION);
+ const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
+ if (!is_empty) checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+
+ uint64_t theta = theta_constants::MAX_THETA;
+ uint32_t num_entries = 0;
+ if (!is_empty) {
+ if (preamble_longs == 1) {
+ num_entries = 1;
+ } else {
+ is.read(reinterpret_cast<char*>(&num_entries), sizeof(num_entries));
+ uint32_t unused32;
+ is.read(reinterpret_cast<char*>(&unused32), sizeof(unused32));
+ if (preamble_longs > 2) {
+ is.read(reinterpret_cast<char*>(&theta), sizeof(theta));
+ }
+ }
+ }
+ std::vector<uint64_t, A> entries(num_entries, 0, allocator);
+ if (!is_empty) is.read(reinterpret_cast<char*>(entries.data()), sizeof(uint64_t) * entries.size());
+
+ const bool is_ordered = flags_byte & (1 << flags::IS_ORDERED);
+ if (!is.good()) throw std::runtime_error("error reading from std::istream");
+ return compact_theta_sketch_experimental(is_empty, is_ordered, seed_hash, theta, std::move(entries));
+}
+
+template<typename A>
+compact_theta_sketch_experimental<A> compact_theta_sketch_experimental<A>::deserialize(const void* bytes, size_t size, uint64_t seed, const A& allocator) {
+ ensure_minimum_memory(size, 8);
+ const char* ptr = static_cast<const char*>(bytes);
+ const char* base = ptr;
+ uint8_t preamble_longs;
+ ptr += copy_from_mem(ptr, &preamble_longs, sizeof(preamble_longs));
+ uint8_t serial_version;
+ ptr += copy_from_mem(ptr, &serial_version, sizeof(serial_version));
+ uint8_t type;
+ ptr += copy_from_mem(ptr, &type, sizeof(type));
+ uint16_t unused16;
+ ptr += copy_from_mem(ptr, &unused16, sizeof(unused16));
+ uint8_t flags_byte;
+ ptr += copy_from_mem(ptr, &flags_byte, sizeof(flags_byte));
+ uint16_t seed_hash;
+ ptr += copy_from_mem(ptr, &seed_hash, sizeof(seed_hash));
+ checker<true>::check_sketch_type(type, SKETCH_TYPE);
+ checker<true>::check_serial_version(serial_version, SERIAL_VERSION);
+ const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
+ if (!is_empty) checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+
+ uint64_t theta = theta_constants::MAX_THETA;
+ uint32_t num_entries = 0;
+ if (!is_empty) {
+ if (preamble_longs == 1) {
+ num_entries = 1;
+ } else {
+ ensure_minimum_memory(size, 8); // read the first prelong before this method
+ ptr += copy_from_mem(ptr, &num_entries, sizeof(num_entries));
+ uint32_t unused32;
+ ptr += copy_from_mem(ptr, &unused32, sizeof(unused32));
+ if (preamble_longs > 2) {
+ ensure_minimum_memory(size, (preamble_longs - 1) << 3);
+ ptr += copy_from_mem(ptr, &theta, sizeof(theta));
+ }
+ }
+ }
+ const size_t entries_size_bytes = sizeof(uint64_t) * num_entries;
+ check_memory_size(ptr - base + entries_size_bytes, size);
+ std::vector<uint64_t, A> entries(num_entries, 0, allocator);
+ if (!is_empty) ptr += copy_from_mem(ptr, entries.data(), entries_size_bytes);
+
+ const bool is_ordered = flags_byte & (1 << flags::IS_ORDERED);
+ return compact_theta_sketch_experimental(is_empty, is_ordered, seed_hash, theta, std::move(entries));
+}
+
+} /* namespace datasketches */
diff --git a/tuple/include/theta_union_base.hpp b/tuple/include/theta_union_base.hpp
new file mode 100644
index 0000000..3072630
--- /dev/null
+++ b/tuple/include/theta_union_base.hpp
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef THETA_UNION_BASE_HPP_
+#define THETA_UNION_BASE_HPP_
+
+#include "theta_update_sketch_base.hpp"
+
+namespace datasketches {
+
+template<
+ typename Entry,
+ typename ExtractKey,
+ typename Policy,
+ typename Sketch,
+ typename CompactSketch,
+ typename Allocator = std::allocator<Entry>
+>
+class theta_union_base {
+public:
+ using hash_table = theta_update_sketch_base<Entry, ExtractKey, Allocator>;
+ using resize_factor = typename hash_table::resize_factor;
+ using comparator = compare_by_key<ExtractKey>;
+
+ theta_union_base(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta, uint64_t seed, const Policy& policy, const Allocator& allocator);
+
+ template<typename FwdSketch>
+ void update(FwdSketch&& sketch);
+
+ CompactSketch get_result(bool ordered = true) const;
+
+ const Policy& get_policy() const;
+
+private:
+ Policy policy_;
+ hash_table table_;
+ uint64_t union_theta_;
+};
+
+} /* namespace datasketches */
+
+#include "theta_union_base_impl.hpp"
+
+#endif
diff --git a/tuple/include/theta_union_base_impl.hpp b/tuple/include/theta_union_base_impl.hpp
new file mode 100644
index 0000000..a86ba3e
--- /dev/null
+++ b/tuple/include/theta_union_base_impl.hpp
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <algorithm>
+
+#include "conditional_forward.hpp"
+
+namespace datasketches {
+
+template<typename EN, typename EK, typename P, typename S, typename CS, typename A>
+theta_union_base<EN, EK, P, S, CS, A>::theta_union_base(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf,
+ uint64_t theta, uint64_t seed, const P& policy, const A& allocator):
+policy_(policy),
+table_(lg_cur_size, lg_nom_size, rf, theta, seed, allocator),
+union_theta_(table_.theta_)
+{}
+
+template<typename EN, typename EK, typename P, typename S, typename CS, typename A>
+template<typename SS>
+void theta_union_base<EN, EK, P, S, CS, A>::update(SS&& sketch) {
+ if (sketch.is_empty()) return;
+ if (sketch.get_seed_hash() != compute_seed_hash(table_.seed_)) throw std::invalid_argument("seed hash mismatch");
+ table_.is_empty_ = false;
+ if (sketch.get_theta64() < union_theta_) union_theta_ = sketch.get_theta64();
+ for (auto& entry: sketch) {
+ const uint64_t hash = EK()(entry);
+ if (hash < union_theta_) {
+ auto result = table_.find(hash);
+ if (!result.second) {
+ table_.insert(result.first, conditional_forward<SS>(entry));
+ } else {
+ policy_(*result.first, conditional_forward<SS>(entry));
+ }
+ } else {
+ if (sketch.is_ordered()) break; // early stop
+ }
+ }
+ if (table_.theta_ < union_theta_) union_theta_ = table_.theta_;
+}
+
+template<typename EN, typename EK, typename P, typename S, typename CS, typename A>
+CS theta_union_base<EN, EK, P, S, CS, A>::get_result(bool ordered) const {
+ std::vector<EN, A> entries(table_.allocator_);
+ if (table_.is_empty_) return CS(true, true, compute_seed_hash(table_.seed_), union_theta_, std::move(entries));
+ entries.reserve(table_.num_entries_);
+ uint64_t theta = std::min(union_theta_, table_.theta_);
+ const uint32_t nominal_num = 1 << table_.lg_nom_size_;
+ if (union_theta_ >= theta && table_.num_entries_ <= nominal_num) {
+ std::copy_if(table_.begin(), table_.end(), std::back_inserter(entries), key_not_zero<EN, EK>());
+ } else {
+ std::copy_if(table_.begin(), table_.end(), std::back_inserter(entries), key_not_zero_less_than<uint64_t, EN, EK>(theta));
+ if (entries.size() > nominal_num) {
+ std::nth_element(entries.begin(), entries.begin() + nominal_num, entries.end(), comparator());
+ theta = EK()(entries[nominal_num]);
+ entries.erase(entries.begin() + nominal_num, entries.end());
+ entries.shrink_to_fit();
+ }
+ }
+ if (ordered) std::sort(entries.begin(), entries.end(), comparator());
+ return CS(table_.is_empty_, ordered, compute_seed_hash(table_.seed_), theta, std::move(entries));
+}
+
+template<typename EN, typename EK, typename P, typename S, typename CS, typename A>
+const P& theta_union_base<EN, EK, P, S, CS, A>::get_policy() const {
+ return policy_;
+}
+
+} /* namespace datasketches */
diff --git a/tuple/include/theta_union_experimental.hpp b/tuple/include/theta_union_experimental.hpp
new file mode 100644
index 0000000..0849f70
--- /dev/null
+++ b/tuple/include/theta_union_experimental.hpp
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef THETA_UNION_EXPERIMENTAL_HPP_
+#define THETA_UNION_EXPERIMENTAL_HPP_
+
+#include "serde.hpp"
+#include "tuple_sketch.hpp"
+#include "theta_union_base.hpp"
+#include "theta_sketch_experimental.hpp"
+
+namespace datasketches {
+
+// experimental theta union derived from the same base as tuple union
+
+template<typename Allocator = std::allocator<uint64_t>>
+class theta_union_experimental {
+public:
+ using Entry = uint64_t;
+ using ExtractKey = trivial_extract_key;
+ using Sketch = theta_sketch_experimental<Allocator>;
+ using CompactSketch = compact_theta_sketch_experimental<Allocator>;
+ using resize_factor = theta_constants::resize_factor;
+
+ struct pass_through_policy {
+ uint64_t operator()(uint64_t internal_entry, uint64_t incoming_entry) const {
+ unused(incoming_entry);
+ return internal_entry;
+ }
+ };
+ using State = theta_union_base<Entry, ExtractKey, pass_through_policy, Sketch, CompactSketch, Allocator>;
+
+ // No constructor here. Use builder instead.
+ class builder;
+
+ /**
+ * This method is to update the union with a given sketch
+ * @param sketch to update the union with
+ */
+ void update(const Sketch& sketch);
+
+ /**
+ * This method produces a copy of the current state of the union as a compact sketch.
+ * @param ordered optional flag to specify if ordered sketch should be produced
+ * @return the result of the union
+ */
+ CompactSketch get_result(bool ordered = true) const;
+
+private:
+ State state_;
+
+ // for builder
+ theta_union_experimental(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta, uint64_t seed, const Allocator& allocator);
+};
+
+template<typename A>
+class theta_union_experimental<A>::builder: public theta_base_builder<builder, A> {
+public:
+ builder(const A& allocator = A());
+
+ /**
+ * This is to create an instance of the union with predefined parameters.
+ * @return an instance of the union
+ */
+ theta_union_experimental<A> build() const;
+};
+
+} /* namespace datasketches */
+
+#include "theta_union_experimental_impl.hpp"
+
+#endif
diff --git a/tuple/include/theta_union_experimental_impl.hpp b/tuple/include/theta_union_experimental_impl.hpp
new file mode 100644
index 0000000..f80afe4
--- /dev/null
+++ b/tuple/include/theta_union_experimental_impl.hpp
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+namespace datasketches {
+
+template<typename A>
+theta_union_experimental<A>::theta_union_experimental(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta, uint64_t seed, const A& allocator):
+state_(lg_cur_size, lg_nom_size, rf, theta, seed, pass_through_policy(), allocator)
+{}
+
+template<typename A>
+void theta_union_experimental<A>::update(const Sketch& sketch) {
+ state_.update(sketch);
+}
+
+template<typename A>
+auto theta_union_experimental<A>::get_result(bool ordered) const -> CompactSketch {
+ return state_.get_result(ordered);
+}
+
+template<typename A>
+theta_union_experimental<A>::builder::builder(const A& allocator): theta_base_builder<builder, A>(allocator) {}
+
+template<typename A>
+auto theta_union_experimental<A>::builder::build() const -> theta_union_experimental {
+ return theta_union_experimental(
+ this->starting_sub_multiple(this->lg_k_ + 1, this->MIN_LG_K, static_cast<uint8_t>(this->rf_)),
+ this->lg_k_, this->rf_, this->starting_theta(), this->seed_, this->allocator_);
+}
+
+} /* namespace datasketches */
diff --git a/tuple/include/theta_update_sketch_base.hpp b/tuple/include/theta_update_sketch_base.hpp
new file mode 100644
index 0000000..425a8bd
--- /dev/null
+++ b/tuple/include/theta_update_sketch_base.hpp
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef THETA_UPDATE_SKETCH_BASE_HPP_
+#define THETA_UPDATE_SKETCH_BASE_HPP_
+
+#include <vector>
+#include <climits>
+#include <cmath>
+
+#include "common_defs.hpp"
+#include "MurmurHash3.h"
+#include "theta_comparators.hpp"
+#include "theta_constants.hpp"
+
+namespace datasketches {
+
+template<
+ typename Entry,
+ typename ExtractKey,
+ typename Allocator = std::allocator<Entry>
+>
+struct theta_update_sketch_base {
+ using resize_factor = theta_constants::resize_factor;
+ using comparator = compare_by_key<ExtractKey>;
+
+ theta_update_sketch_base(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta,
+ uint64_t seed, const Allocator& allocator, bool is_empty = true);
+ theta_update_sketch_base(const theta_update_sketch_base& other);
+ theta_update_sketch_base(theta_update_sketch_base&& other) noexcept;
+ ~theta_update_sketch_base();
+ theta_update_sketch_base& operator=(const theta_update_sketch_base& other);
+ theta_update_sketch_base& operator=(theta_update_sketch_base&& other);
+
+ using iterator = Entry*;
+
+ inline uint64_t hash_and_screen(const void* data, size_t length);
+
+ inline std::pair<iterator, bool> find(uint64_t key) const;
+
+ template<typename FwdEntry>
+ inline void insert(iterator it, FwdEntry&& entry);
+
+ iterator begin() const;
+ iterator end() const;
+
+ // resize threshold = 0.5 tuned for speed
+ static constexpr double RESIZE_THRESHOLD = 0.5;
+ // hash table rebuild threshold = 15/16
+ static constexpr double REBUILD_THRESHOLD = 15.0 / 16.0;
+
+ static constexpr uint8_t STRIDE_HASH_BITS = 7;
+ static constexpr uint32_t STRIDE_MASK = (1 << STRIDE_HASH_BITS) - 1;
+
+ Allocator allocator_;
+ bool is_empty_;
+ uint8_t lg_cur_size_;
+ uint8_t lg_nom_size_;
+ resize_factor rf_;
+ uint32_t num_entries_;
+ uint64_t theta_;
+ uint64_t seed_;
+ Entry* entries_;
+
+ void resize();
+ void rebuild();
+ void trim();
+
+ static inline uint32_t get_capacity(uint8_t lg_cur_size, uint8_t lg_nom_size);
+ static inline uint32_t get_stride(uint64_t key, uint8_t lg_size);
+ static void consolidate_non_empty(Entry* entries, size_t size, size_t num);
+};
+
+// builder
+
+template<typename Derived, typename Allocator>
+class theta_base_builder {
+public:
+ using resize_factor = theta_constants::resize_factor;
+ static const uint8_t MIN_LG_K = theta_constants::MIN_LG_K;
+ static const uint8_t MAX_LG_K = theta_constants::MAX_LG_K;
+ static const uint8_t DEFAULT_LG_K = 12;
+ static const resize_factor DEFAULT_RESIZE_FACTOR = resize_factor::X8;
+
+ /**
+ * Creates and instance of the builder with default parameters.
+ */
+ theta_base_builder(const Allocator& allocator);
+
+ /**
+ * Set log2(k), where k is a nominal number of entries in the sketch
+ * @param lg_k base 2 logarithm of nominal number of entries
+ * @return this builder
+ */
+ Derived& set_lg_k(uint8_t lg_k);
+
+ /**
+ * Set resize factor for the internal hash table (defaults to 8)
+ * @param rf resize factor
+ * @return this builder
+ */
+ Derived& set_resize_factor(resize_factor rf);
+
+ /**
+ * Set sampling probability (initial theta). The default is 1, so the sketch retains
+ * all entries until it reaches the limit, at which point it goes into the estimation mode
+ * and reduces the effective sampling probability (theta) as necessary.
+ * @param p sampling probability
+ * @return this builder
+ */
+ Derived& set_p(float p);
+
+ /**
+ * Set the seed for the hash function. Should be used carefully if needed.
+ * Sketches produced with different seed are not compatible
+ * and cannot be mixed in set operations.
+ * @param seed hash seed
+ * @return this builder
+ */
+ Derived& set_seed(uint64_t seed);
+
+protected:
+ Allocator allocator_;
+ uint8_t lg_k_;
+ resize_factor rf_;
+ float p_;
+ uint64_t seed_;
+
+ uint64_t starting_theta() const;
+ uint8_t starting_lg_size() const;
+ static uint8_t starting_sub_multiple(uint8_t lg_tgt, uint8_t lg_min, uint8_t lg_rf);
+};
+
+// key extractors
+
+struct trivial_extract_key {
+ template<typename T>
+ auto operator()(T&& entry) const -> decltype(std::forward<T>(entry)) {
+ return std::forward<T>(entry);
+ }
+};
+
+template<typename K, typename V>
+struct pair_extract_key {
+ K& operator()(std::pair<K, V>& entry) const {
+ return entry.first;
+ }
+ const K& operator()(const std::pair<K, V>& entry) const {
+ return entry.first;
+ }
+};
+
+// not zero
+
+template<typename Entry, typename ExtractKey>
+class key_not_zero {
+public:
+ bool operator()(const Entry& entry) const {
+ return ExtractKey()(entry) != 0;
+ }
+};
+
+template<typename Key, typename Entry, typename ExtractKey>
+class key_not_zero_less_than {
+public:
+ explicit key_not_zero_less_than(const Key& key): key(key) {}
+ bool operator()(const Entry& entry) const {
+ return ExtractKey()(entry) != 0 && ExtractKey()(entry) < this->key;
+ }
+private:
+ Key key;
+};
+
+// MurMur3 hash functions
+
+static inline uint64_t compute_hash(const void* data, size_t length, uint64_t seed) {
+ HashState hashes;
+ MurmurHash3_x64_128(data, length, seed, hashes);
+ return (hashes.h1 >> 1); // Java implementation does unsigned shift >>> to make values positive
+}
+
+static inline uint16_t compute_seed_hash(uint64_t seed) {
+ HashState hashes;
+ MurmurHash3_x64_128(&seed, sizeof(seed), 0, hashes);
+ return hashes.h1;
+}
+
+// iterators
+
+template<typename Entry, typename ExtractKey>
+class theta_iterator: public std::iterator<std::input_iterator_tag, Entry> {
+public:
+ theta_iterator(Entry* entries, uint32_t size, uint32_t index);
+ theta_iterator& operator++();
+ theta_iterator operator++(int);
+ bool operator==(const theta_iterator& other) const;
+ bool operator!=(const theta_iterator& other) const;
+ Entry& operator*() const;
+
+private:
+ Entry* entries_;
+ uint32_t size_;
+ uint32_t index_;
+};
+
+template<typename Entry, typename ExtractKey>
+class theta_const_iterator: public std::iterator<std::input_iterator_tag, Entry> {
+public:
+ theta_const_iterator(const Entry* entries, uint32_t size, uint32_t index);
+ theta_const_iterator& operator++();
+ theta_const_iterator operator++(int);
+ bool operator==(const theta_const_iterator& other) const;
+ bool operator!=(const theta_const_iterator& other) const;
+ const Entry& operator*() const;
+
+private:
+ const Entry* entries_;
+ uint32_t size_;
+ uint32_t index_;
+};
+
+// double value canonicalization for compatibility with Java
+static inline int64_t canonical_double(double value) {
+ union {
+ int64_t long_value;
+ double double_value;
+ } long_double_union;
+
+ if (value == 0.0) {
+ long_double_union.double_value = 0.0; // canonicalize -0.0 to 0.0
+ } else if (std::isnan(value)) {
+ long_double_union.long_value = 0x7ff8000000000000L; // canonicalize NaN using value from Java's Double.doubleToLongBits()
+ } else {
+ long_double_union.double_value = value;
+ }
+ return long_double_union.long_value;
+}
+
+} /* namespace datasketches */
+
+#include "theta_update_sketch_base_impl.hpp"
+
+#endif
diff --git a/tuple/include/theta_update_sketch_base_impl.hpp b/tuple/include/theta_update_sketch_base_impl.hpp
new file mode 100644
index 0000000..8fdb3d8
--- /dev/null
+++ b/tuple/include/theta_update_sketch_base_impl.hpp
@@ -0,0 +1,389 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <iostream>
+#include <sstream>
+#include <algorithm>
+
+namespace datasketches {
+
+template<typename EN, typename EK, typename A>
+theta_update_sketch_base<EN, EK, A>::theta_update_sketch_base(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta, uint64_t seed, const A& allocator, bool is_empty):
+allocator_(allocator),
+is_empty_(is_empty),
+lg_cur_size_(lg_cur_size),
+lg_nom_size_(lg_nom_size),
+rf_(rf),
+num_entries_(0),
+theta_(theta),
+seed_(seed),
+entries_(nullptr)
+{
+ if (lg_cur_size > 0) {
+ const size_t size = 1 << lg_cur_size;
+ entries_ = allocator_.allocate(size);
+ for (size_t i = 0; i < size; ++i) EK()(entries_[i]) = 0;
+ }
+}
+
+template<typename EN, typename EK, typename A>
+theta_update_sketch_base<EN, EK, A>::theta_update_sketch_base(const theta_update_sketch_base& other):
+allocator_(other.allocator_),
+is_empty_(other.is_empty_),
+lg_cur_size_(other.lg_cur_size_),
+lg_nom_size_(other.lg_nom_size_),
+rf_(other.rf_),
+num_entries_(other.num_entries_),
+theta_(other.theta_),
+seed_(other.seed_),
+entries_(nullptr)
+{
+ if (other.entries_ != nullptr) {
+ const size_t size = 1 << lg_cur_size_;
+ entries_ = allocator_.allocate(size);
+ for (size_t i = 0; i < size; ++i) {
+ if (EK()(other.entries_[i]) != 0) {
+ new (&entries_[i]) EN(other.entries_[i]);
+ } else {
+ EK()(entries_[i]) = 0;
+ }
+ }
+ }
+}
+
+template<typename EN, typename EK, typename A>
+theta_update_sketch_base<EN, EK, A>::theta_update_sketch_base(theta_update_sketch_base&& other) noexcept:
+allocator_(other.allocator_),
+is_empty_(other.is_empty_),
+lg_cur_size_(other.lg_cur_size_),
+lg_nom_size_(other.lg_nom_size_),
+rf_(other.rf_),
+num_entries_(other.num_entries_),
+theta_(other.theta_),
+seed_(other.seed_),
+entries_(other.entries_)
+{
+ other.entries_ = nullptr;
+}
+
+template<typename EN, typename EK, typename A>
+theta_update_sketch_base<EN, EK, A>::~theta_update_sketch_base()
+{
+ if (entries_ != nullptr) {
+ const size_t size = 1 << lg_cur_size_;
+ for (size_t i = 0; i < size; ++i) {
+ if (EK()(entries_[i]) != 0) entries_[i].~EN();
+ }
+ allocator_.deallocate(entries_, size);
+ }
+}
+
+template<typename EN, typename EK, typename A>
+theta_update_sketch_base<EN, EK, A>& theta_update_sketch_base<EN, EK, A>::operator=(const theta_update_sketch_base& other) {
+ theta_update_sketch_base<EN, EK, A> copy(other);
+ std::swap(allocator_, copy.allocator_);
+ std::swap(is_empty_, copy.is_empty_);
+ std::swap(lg_cur_size_, copy.lg_cur_size_);
+ std::swap(lg_nom_size_, copy.lg_nom_size_);
+ std::swap(rf_, copy.rf_);
+ std::swap(num_entries_, copy.num_entries_);
+ std::swap(theta_, copy.theta_);
+ std::swap(seed_, copy.seed_);
+ std::swap(entries_, copy.entries_);
+ return *this;
+}
+
+template<typename EN, typename EK, typename A>
+theta_update_sketch_base<EN, EK, A>& theta_update_sketch_base<EN, EK, A>::operator=(theta_update_sketch_base&& other) {
+ std::swap(allocator_, other.allocator_);
+ std::swap(is_empty_, other.is_empty_);
+ std::swap(lg_cur_size_, other.lg_cur_size_);
+ std::swap(lg_nom_size_, other.lg_nom_size_);
+ std::swap(rf_, other.rf_);
+ std::swap(num_entries_, other.num_entries_);
+ std::swap(theta_, other.theta_);
+ std::swap(seed_, other.seed_);
+ std::swap(entries_, other.entries_);
+ return *this;
+}
+
+template<typename EN, typename EK, typename A>
+uint64_t theta_update_sketch_base<EN, EK, A>::hash_and_screen(const void* data, size_t length) {
+ is_empty_ = false;
+ const uint64_t hash = compute_hash(data, length, seed_);
+ if (hash >= theta_) return 0; // hash == 0 is reserved to mark empty slots in the table
+ return hash;
+}
+
+template<typename EN, typename EK, typename A>
+auto theta_update_sketch_base<EN, EK, A>::find(uint64_t key) const -> std::pair<iterator, bool> {
+ const size_t size = 1 << lg_cur_size_;
+ const size_t mask = size - 1;
+ const uint32_t stride = get_stride(key, lg_cur_size_);
+ uint32_t index = static_cast<uint32_t>(key) & mask;
+ // search for duplicate or zero
+ const uint32_t loop_index = index;
+ do {
+ const uint64_t probe = EK()(entries_[index]);
+ if (probe == 0) {
+ return std::pair<iterator, bool>(&entries_[index], false);
+ } else if (probe == key) {
+ return std::pair<iterator, bool>(&entries_[index], true);
+ }
+ index = (index + stride) & mask;
+ } while (index != loop_index);
+ throw std::logic_error("key not found and no empty slots!");
+}
+
+template<typename EN, typename EK, typename A>
+template<typename Fwd>
+void theta_update_sketch_base<EN, EK, A>::insert(iterator it, Fwd&& entry) {
+ new (it) EN(std::forward<Fwd>(entry));
+ ++num_entries_;
+ if (num_entries_ > get_capacity(lg_cur_size_, lg_nom_size_)) {
+ if (lg_cur_size_ <= lg_nom_size_) {
+ resize();
+ } else {
+ rebuild();
+ }
+ }
+}
+
+template<typename EN, typename EK, typename A>
+auto theta_update_sketch_base<EN, EK, A>::begin() const -> iterator {
+ return entries_;
+}
+
+template<typename EN, typename EK, typename A>
+auto theta_update_sketch_base<EN, EK, A>::end() const -> iterator {
+ return &entries_[1 << lg_cur_size_];
+}
+
+template<typename EN, typename EK, typename A>
+uint32_t theta_update_sketch_base<EN, EK, A>::get_capacity(uint8_t lg_cur_size, uint8_t lg_nom_size) {
+ const double fraction = (lg_cur_size <= lg_nom_size) ? RESIZE_THRESHOLD : REBUILD_THRESHOLD;
+ return std::floor(fraction * (1 << lg_cur_size));
+}
+
+template<typename EN, typename EK, typename A>
+uint32_t theta_update_sketch_base<EN, EK, A>::get_stride(uint64_t key, uint8_t lg_size) {
+ // odd and independent of index assuming lg_size lowest bits of the key were used for the index
+ return (2 * static_cast<uint32_t>((key >> lg_size) & STRIDE_MASK)) + 1;
+}
+
+template<typename EN, typename EK, typename A>
+void theta_update_sketch_base<EN, EK, A>::resize() {
+ const size_t old_size = 1 << lg_cur_size_;
+ const uint8_t lg_tgt_size = lg_nom_size_ + 1;
+ const uint8_t factor = std::max(1, std::min(static_cast<int>(rf_), lg_tgt_size - lg_cur_size_));
+ lg_cur_size_ += factor;
+ const size_t new_size = 1 << lg_cur_size_;
+ EN* old_entries = entries_;
+ entries_ = allocator_.allocate(new_size);
+ for (size_t i = 0; i < new_size; ++i) EK()(entries_[i]) = 0;
+ num_entries_ = 0;
+ for (size_t i = 0; i < old_size; ++i) {
+ const uint64_t key = EK()(old_entries[i]);
+ if (key != 0) {
+ insert(find(key).first, std::move(old_entries[i])); // consider a special insert with no comparison
+ old_entries[i].~EN();
+ }
+ }
+ allocator_.deallocate(old_entries, old_size);
+}
+
+// assumes number of entries > nominal size
+template<typename EN, typename EK, typename A>
+void theta_update_sketch_base<EN, EK, A>::rebuild() {
+ const size_t size = 1 << lg_cur_size_;
+ const uint32_t nominal_size = 1 << lg_nom_size_;
+
+ // empty entries have uninitialized payloads
+ // TODO: avoid this for empty or trivial payloads (arithmetic types)
+ consolidate_non_empty(entries_, size, num_entries_);
+
+ std::nth_element(entries_, entries_ + nominal_size, entries_ + num_entries_, comparator());
+ this->theta_ = EK()(entries_[nominal_size]);
+ EN* old_entries = entries_;
+ const size_t num_old_entries = num_entries_;
+ entries_ = allocator_.allocate(size);
+ for (size_t i = 0; i < size; ++i) EK()(entries_[i]) = 0;
+ num_entries_ = 0;
+ // relies on consolidating non-empty entries to the front
+ for (size_t i = 0; i < nominal_size; ++i) {
+ insert(find(EK()(old_entries[i])).first, std::move(old_entries[i])); // consider a special insert with no comparison
+ old_entries[i].~EN();
+ }
+ for (size_t i = nominal_size; i < num_old_entries; ++i) old_entries[i].~EN();
+ allocator_.deallocate(old_entries, size);
+}
+
+template<typename EN, typename EK, typename A>
+void theta_update_sketch_base<EN, EK, A>::trim() {
+ if (num_entries_ > static_cast<uint32_t>(1 << lg_nom_size_)) rebuild();
+}
+
+template<typename EN, typename EK, typename A>
+void theta_update_sketch_base<EN, EK, A>::consolidate_non_empty(EN* entries, size_t size, size_t num) {
+ // find the first empty slot
+ size_t i = 0;
+ while (i < size) {
+ if (EK()(entries[i]) == 0) break;
+ ++i;
+ }
+ // scan the rest and move non-empty entries to the front
+ for (size_t j = i + 1; j < size; ++j) {
+ if (EK()(entries[j]) != 0) {
+ new (&entries[i]) EN(std::move(entries[j]));
+ entries[j].~EN();
+ EK()(entries[j]) = 0;
+ ++i;
+ if (i == num) break;
+ }
+ }
+}
+
+// builder
+
+template<typename Derived, typename Allocator>
+theta_base_builder<Derived, Allocator>::theta_base_builder(const Allocator& allocator):
+allocator_(allocator), lg_k_(DEFAULT_LG_K), rf_(DEFAULT_RESIZE_FACTOR), p_(1), seed_(DEFAULT_SEED) {}
+
+template<typename Derived, typename Allocator>
+Derived& theta_base_builder<Derived, Allocator>::set_lg_k(uint8_t lg_k) {
+ if (lg_k < MIN_LG_K) {
+ throw std::invalid_argument("lg_k must not be less than " + std::to_string(MIN_LG_K) + ": " + std::to_string(lg_k));
+ }
+ if (lg_k > MAX_LG_K) {
+ throw std::invalid_argument("lg_k must not be greater than " + std::to_string(MAX_LG_K) + ": " + std::to_string(lg_k));
+ }
+ lg_k_ = lg_k;
+ return static_cast<Derived&>(*this);
+}
+
+template<typename Derived, typename Allocator>
+Derived& theta_base_builder<Derived, Allocator>::set_resize_factor(resize_factor rf) {
+ rf_ = rf;
+ return static_cast<Derived&>(*this);
+}
+
+template<typename Derived, typename Allocator>
+Derived& theta_base_builder<Derived, Allocator>::set_p(float p) {
+ if (p <= 0 || p > 1) throw std::invalid_argument("sampling probability must be between 0 and 1");
+ p_ = p;
+ return static_cast<Derived&>(*this);
+}
+
+template<typename Derived, typename Allocator>
+Derived& theta_base_builder<Derived, Allocator>::set_seed(uint64_t seed) {
+ seed_ = seed;
+ return static_cast<Derived&>(*this);
+}
+
+template<typename Derived, typename Allocator>
+uint64_t theta_base_builder<Derived, Allocator>::starting_theta() const {
+ if (p_ < 1) return theta_constants::MAX_THETA * p_;
+ return theta_constants::MAX_THETA;
+}
+
+template<typename Derived, typename Allocator>
+uint8_t theta_base_builder<Derived, Allocator>::starting_lg_size() const {
+ return starting_sub_multiple(lg_k_ + 1, MIN_LG_K, static_cast<uint8_t>(rf_));
+}
+
+template<typename Derived, typename Allocator>
+uint8_t theta_base_builder<Derived, Allocator>::starting_sub_multiple(uint8_t lg_tgt, uint8_t lg_min, uint8_t lg_rf) {
+ return (lg_tgt <= lg_min) ? lg_min : (lg_rf == 0) ? lg_tgt : ((lg_tgt - lg_min) % lg_rf) + lg_min;
+}
+
+// iterator
+
+template<typename Entry, typename ExtractKey>
+theta_iterator<Entry, ExtractKey>::theta_iterator(Entry* entries, uint32_t size, uint32_t index):
+entries_(entries), size_(size), index_(index) {
+ while (index_ < size_ && ExtractKey()(entries_[index_]) == 0) ++index_;
+}
+
+template<typename Entry, typename ExtractKey>
+auto theta_iterator<Entry, ExtractKey>::operator++() -> theta_iterator& {
+ ++index_;
+ while (index_ < size_ && ExtractKey()(entries_[index_]) == 0) ++index_;
+ return *this;
+}
+
+template<typename Entry, typename ExtractKey>
+auto theta_iterator<Entry, ExtractKey>::operator++(int) -> theta_iterator {
+ theta_iterator tmp(*this);
+ operator++();
+ return tmp;
+}
+
+template<typename Entry, typename ExtractKey>
+bool theta_iterator<Entry, ExtractKey>::operator!=(const theta_iterator& other) const {
+ return index_ != other.index_;
+}
+
+template<typename Entry, typename ExtractKey>
+bool theta_iterator<Entry, ExtractKey>::operator==(const theta_iterator& other) const {
+ return index_ == other.index_;
+}
+
+template<typename Entry, typename ExtractKey>
+auto theta_iterator<Entry, ExtractKey>::operator*() const -> Entry& {
+ return entries_[index_];
+}
+
+// const iterator
+
+template<typename Entry, typename ExtractKey>
+theta_const_iterator<Entry, ExtractKey>::theta_const_iterator(const Entry* entries, uint32_t size, uint32_t index):
+entries_(entries), size_(size), index_(index) {
+ while (index_ < size_ && ExtractKey()(entries_[index_]) == 0) ++index_;
+}
+
+template<typename Entry, typename ExtractKey>
+auto theta_const_iterator<Entry, ExtractKey>::operator++() -> theta_const_iterator& {
+ ++index_;
+ while (index_ < size_ && ExtractKey()(entries_[index_]) == 0) ++index_;
+ return *this;
+}
+
+template<typename Entry, typename ExtractKey>
+auto theta_const_iterator<Entry, ExtractKey>::operator++(int) -> theta_const_iterator {
+ theta_const_iterator tmp(*this);
+ operator++();
+ return tmp;
+}
+
+template<typename Entry, typename ExtractKey>
+bool theta_const_iterator<Entry, ExtractKey>::operator!=(const theta_const_iterator& other) const {
+ return index_ != other.index_;
+}
+
+template<typename Entry, typename ExtractKey>
+bool theta_const_iterator<Entry, ExtractKey>::operator==(const theta_const_iterator& other) const {
+ return index_ == other.index_;
+}
+
+template<typename Entry, typename ExtractKey>
+auto theta_const_iterator<Entry, ExtractKey>::operator*() const -> const Entry& {
+ return entries_[index_];
+}
+
+} /* namespace datasketches */
diff --git a/tuple/include/tuple_a_not_b.hpp b/tuple/include/tuple_a_not_b.hpp
new file mode 100644
index 0000000..5c258fb
--- /dev/null
+++ b/tuple/include/tuple_a_not_b.hpp
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef TUPLE_A_NOT_B_HPP_
+#define TUPLE_A_NOT_B_HPP_
+
+#include "tuple_sketch.hpp"
+#include "theta_set_difference_base.hpp"
+
+namespace datasketches {
+
+template<
+ typename Summary,
+ typename Allocator = std::allocator<Summary>
+>
+class tuple_a_not_b {
+public:
+ using Entry = std::pair<uint64_t, Summary>;
+ using ExtractKey = pair_extract_key<uint64_t, Summary>;
+ using CompactSketch = compact_tuple_sketch<Summary, Allocator>;
+ using AllocEntry = typename std::allocator_traits<Allocator>::template rebind_alloc<Entry>;
+ using State = theta_set_difference_base<Entry, ExtractKey, CompactSketch, AllocEntry>;
+
+ explicit tuple_a_not_b(uint64_t seed = DEFAULT_SEED, const Allocator& allocator = Allocator());
+
+ /**
+ * Computes the a-not-b set operation given two sketches.
+ * @return the result of a-not-b
+ */
+ template<typename FwdSketch, typename Sketch>
+ CompactSketch compute(FwdSketch&& a, const Sketch& b, bool ordered = true) const;
+
+private:
+ State state_;
+};
+
+} /* namespace datasketches */
+
+#include "tuple_a_not_b_impl.hpp"
+
+#endif
diff --git a/tuple/include/tuple_a_not_b_impl.hpp b/tuple/include/tuple_a_not_b_impl.hpp
new file mode 100644
index 0000000..7d7e85b
--- /dev/null
+++ b/tuple/include/tuple_a_not_b_impl.hpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+namespace datasketches {
+
+template<typename S, typename A>
+tuple_a_not_b<S, A>::tuple_a_not_b(uint64_t seed, const A& allocator):
+state_(seed, allocator)
+{}
+
+template<typename S, typename A>
+template<typename FwdSketch, typename Sketch>
+auto tuple_a_not_b<S, A>::compute(FwdSketch&& a, const Sketch& b, bool ordered) const -> CompactSketch {
+ return state_.compute(std::forward<FwdSketch>(a), b, ordered);
+}
+
+} /* namespace datasketches */
diff --git a/tuple/include/tuple_intersection.hpp b/tuple/include/tuple_intersection.hpp
new file mode 100644
index 0000000..966ea9f
--- /dev/null
+++ b/tuple/include/tuple_intersection.hpp
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef TUPLE_INTERSECTION_HPP_
+#define TUPLE_INTERSECTION_HPP_
+
+#include "tuple_sketch.hpp"
+#include "theta_intersection_base.hpp"
+
+namespace datasketches {
+
+/*
+// for types with defined + operation
+template<typename Summary>
+struct example_intersection_policy {
+ void operator()(Summary& summary, const Summary& other) const {
+ summary += other;
+ }
+ void operator()(Summary& summary, Summary&& other) const {
+ summary += other;
+ }
+};
+*/
+
+template<
+ typename Summary,
+ typename Policy,
+ typename Allocator = std::allocator<Summary>
+>
+class tuple_intersection {
+public:
+ using Entry = std::pair<uint64_t, Summary>;
+ using ExtractKey = pair_extract_key<uint64_t, Summary>;
+ using Sketch = tuple_sketch<Summary, Allocator>;
+ using CompactSketch = compact_tuple_sketch<Summary, Allocator>;
+ using AllocEntry = typename std::allocator_traits<Allocator>::template rebind_alloc<Entry>;
+
+ // reformulate the external policy that operates on Summary
+ // in terms of operations on Entry
+ struct internal_policy {
+ internal_policy(const Policy& policy): policy_(policy) {}
+ void operator()(Entry& internal_entry, const Entry& incoming_entry) const {
+ policy_(internal_entry.second, incoming_entry.second);
+ }
+ void operator()(Entry& internal_entry, Entry&& incoming_entry) const {
+ policy_(internal_entry.second, std::move(incoming_entry.second));
+ }
+ const Policy& get_policy() const { return policy_; }
+ Policy policy_;
+ };
+
+ using State = theta_intersection_base<Entry, ExtractKey, internal_policy, Sketch, CompactSketch, AllocEntry>;
+
+ explicit tuple_intersection(uint64_t seed = DEFAULT_SEED, const Policy& policy = Policy(), const Allocator& allocator = Allocator());
+
+ /**
+ * Updates the intersection with a given sketch.
+ * The intersection can be viewed as starting from the "universe" set, and every update
+ * can reduce the current set to leave the overlapping subset only.
+ * @param sketch represents input set for the intersection
+ */
+ template<typename FwdSketch>
+ void update(FwdSketch&& sketch);
+
+ /**
+ * Produces a copy of the current state of the intersection.
+ * If update() was not called, the state is the infinite "universe",
+ * which is considered an undefined state, and throws an exception.
+ * @param ordered optional flag to specify if ordered sketch should be produced
+ * @return the result of the intersection
+ */
+ CompactSketch get_result(bool ordered = true) const;
+
+ /**
+ * Returns true if the state of the intersection is defined (not infinite "universe").
+ * @return true if the state is valid
+ */
+ bool has_result() const;
+
+protected:
+ State state_;
+};
+
+} /* namespace datasketches */
+
+#include "tuple_intersection_impl.hpp"
+
+#endif
diff --git a/tuple/include/tuple_intersection_impl.hpp b/tuple/include/tuple_intersection_impl.hpp
new file mode 100644
index 0000000..74791c5
--- /dev/null
+++ b/tuple/include/tuple_intersection_impl.hpp
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+namespace datasketches {
+
+template<typename S, typename P, typename A>
+tuple_intersection<S, P, A>::tuple_intersection(uint64_t seed, const P& policy, const A& allocator):
+state_(seed, internal_policy(policy), allocator)
+{}
+
+template<typename S, typename P, typename A>
+template<typename SS>
+void tuple_intersection<S, P, A>::update(SS&& sketch) {
+ state_.update(std::forward<SS>(sketch));
+}
+
+template<typename S, typename P, typename A>
+auto tuple_intersection<S, P, A>::get_result(bool ordered) const -> CompactSketch {
+ return state_.get_result(ordered);
+}
+
+template<typename S, typename P, typename A>
+bool tuple_intersection<S, P, A>::has_result() const {
+ return state_.has_result();
+}
+
+} /* namespace datasketches */
diff --git a/tuple/include/tuple_sketch.hpp b/tuple/include/tuple_sketch.hpp
new file mode 100644
index 0000000..2292937
--- /dev/null
+++ b/tuple/include/tuple_sketch.hpp
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef TUPLE_SKETCH_HPP_
+#define TUPLE_SKETCH_HPP_
+
+#include <string>
+
+#include "serde.hpp"
+#include "theta_update_sketch_base.hpp"
+
+namespace datasketches {
+
+// forward-declarations
+template<typename S, typename A> class tuple_sketch;
+template<typename S, typename U, typename P, typename A> class update_tuple_sketch;
+template<typename S, typename A> class compact_tuple_sketch;
+template<typename A> class theta_sketch_experimental;
+
+template<
+ typename Summary,
+ typename Allocator = std::allocator<Summary>
+>
+class tuple_sketch {
+public:
+ using Entry = std::pair<uint64_t, Summary>;
+ using ExtractKey = pair_extract_key<uint64_t, Summary>;
+ using iterator = theta_iterator<Entry, ExtractKey>;
+ using const_iterator = theta_const_iterator<Entry, ExtractKey>;
+
+ virtual ~tuple_sketch() = default;
+
+ /**
+ * @return allocator
+ */
+ virtual Allocator get_allocator() const = 0;
+
+ /**
+ * @return true if this sketch represents an empty set (not the same as no retained entries!)
+ */
+ virtual bool is_empty() const = 0;
+
+ /**
+ * @return estimate of the distinct count of the input stream
+ */
+ double get_estimate() const;
+
+ /**
+ * Returns the approximate lower error bound given a number of standard deviations.
+ * This parameter is similar to the number of standard deviations of the normal distribution
+ * and corresponds to approximately 67%, 95% and 99% confidence intervals.
+ * @param num_std_devs number of Standard Deviations (1, 2 or 3)
+ * @return the lower bound
+ */
+ double get_lower_bound(uint8_t num_std_devs) const;
+
+ /**
+ * Returns the approximate upper error bound given a number of standard deviations.
+ * This parameter is similar to the number of standard deviations of the normal distribution
+ * and corresponds to approximately 67%, 95% and 99% confidence intervals.
+ * @param num_std_devs number of Standard Deviations (1, 2 or 3)
+ * @return the upper bound
+ */
+ double get_upper_bound(uint8_t num_std_devs) const;
+
+ /**
+ * @return true if the sketch is in estimation mode (as opposed to exact mode)
+ */
+ bool is_estimation_mode() const;
+
+ /**
+ * @return theta as a fraction from 0 to 1 (effective sampling rate)
+ */
+ double get_theta() const;
+
+ /**
+ * @return theta as a positive integer between 0 and LLONG_MAX
+ */
+ virtual uint64_t get_theta64() const = 0;
+
+ /**
+ * @return the number of retained entries in the sketch
+ */
+ virtual uint32_t get_num_retained() const = 0;
+
+ /**
+ * @return hash of the seed that was used to hash the input
+ */
+ virtual uint16_t get_seed_hash() const = 0;
+
+ /**
+ * @return true if retained entries are ordered
+ */
+ virtual bool is_ordered() const = 0;
+
+ /**
+ * Provides a human-readable summary of this sketch as a string
+ * @param print_items if true include the list of items retained by the sketch
+ * @return sketch summary as a string
+ */
+ string<Allocator> to_string(bool print_items = false) const;
+
+ /**
+ * Iterator over entries in this sketch.
+ * @return begin iterator
+ */
+ virtual iterator begin() = 0;
+
+ /**
+ * Iterator pointing past the valid range.
+ * Not to be incremented or dereferenced.
+ * @return end iterator
+ */
+ virtual iterator end() = 0;
+
+ /**
+ * Const iterator over entries in this sketch.
+ * @return begin const iterator
+ */
+ virtual const_iterator begin() const = 0;
+
+ /**
+ * Const iterator pointing past the valid range.
+ * Not to be incremented or dereferenced.
+ * @return end const iterator
+ */
+ virtual const_iterator end() const = 0;
+
+protected:
+ virtual void print_specifics(std::basic_ostream<char>& os) const = 0;
+
+ static uint16_t get_seed_hash(uint64_t seed);
+
+ static void check_sketch_type(uint8_t actual, uint8_t expected);
+ static void check_serial_version(uint8_t actual, uint8_t expected);
+ static void check_seed_hash(uint16_t actual, uint16_t expected);
+};
+
+// update sketch
+
+// for types with defined default constructor and + operation
+template<typename Summary, typename Update>
+struct default_update_policy {
+ Summary create() const {
+ return Summary();
+ }
+ void update(Summary& summary, const Update& update) const {
+ summary += update;
+ }
+};
+
+template<
+ typename Summary,
+ typename Update = Summary,
+ typename Policy = default_update_policy<Summary, Update>,
+ typename Allocator = std::allocator<Summary>
+>
+class update_tuple_sketch: public tuple_sketch<Summary, Allocator> {
+public:
+ using Base = tuple_sketch<Summary, Allocator>;
+ using Entry = typename Base::Entry;
+ using ExtractKey = typename Base::ExtractKey;
+ using iterator = typename Base::iterator;
+ using const_iterator = typename Base::const_iterator;
+ using AllocEntry = typename std::allocator_traits<Allocator>::template rebind_alloc<Entry>;
+ using tuple_map = theta_update_sketch_base<Entry, ExtractKey, AllocEntry>;
+ using resize_factor = typename tuple_map::resize_factor;
+
+ // No constructor here. Use builder instead.
+ class builder;
+
+ update_tuple_sketch(const update_tuple_sketch&) = default;
+ update_tuple_sketch(update_tuple_sketch&&) noexcept = default;
+ virtual ~update_tuple_sketch() = default;
+ update_tuple_sketch& operator=(const update_tuple_sketch&) = default;
+ update_tuple_sketch& operator=(update_tuple_sketch&&) = default;
+
+ virtual Allocator get_allocator() const;
+ virtual bool is_empty() const;
+ virtual bool is_ordered() const;
+ virtual uint64_t get_theta64() const;
+ virtual uint32_t get_num_retained() const;
+ virtual uint16_t get_seed_hash() const;
+
+ /**
+ * @return configured nominal number of entries in the sketch
+ */
+ uint8_t get_lg_k() const;
+
+ /**
+ * @return configured resize factor of the sketch
+ */
+ resize_factor get_rf() const;
+
+ /**
+ * Update this sketch with a given string.
+ * @param value string to update the sketch with
+ */
+ template<typename FwdUpdate>
+ inline void update(const std::string& key, FwdUpdate&& value);
+
+ /**
+ * Update this sketch with a given unsigned 64-bit integer.
+ * @param value uint64_t to update the sketch with
+ */
+ template<typename FwdUpdate>
+ inline void update(uint64_t key, FwdUpdate&& value);
+
+ /**
+ * Update this sketch with a given signed 64-bit integer.
+ * @param value int64_t to update the sketch with
+ */
+ template<typename FwdUpdate>
+ inline void update(int64_t key, FwdUpdate&& value);
+
+ /**
+ * Update this sketch with a given unsigned 32-bit integer.
+ * For compatibility with Java implementation.
+ * @param value uint32_t to update the sketch with
+ */
+ template<typename FwdUpdate>
+ inline void update(uint32_t key, FwdUpdate&& value);
+
+ /**
+ * Update this sketch with a given signed 32-bit integer.
+ * For compatibility with Java implementation.
+ * @param value int32_t to update the sketch with
+ */
+ template<typename FwdUpdate>
+ inline void update(int32_t key, FwdUpdate&& value);
+
+ /**
+ * Update this sketch with a given unsigned 16-bit integer.
+ * For compatibility with Java implementation.
+ * @param value uint16_t to update the sketch with
+ */
+ template<typename FwdUpdate>
+ inline void update(uint16_t key, FwdUpdate&& value);
+
+ /**
+ * Update this sketch with a given signed 16-bit integer.
+ * For compatibility with Java implementation.
+ * @param value int16_t to update the sketch with
+ */
+ template<typename FwdUpdate>
+ inline void update(int16_t key, FwdUpdate&& value);
+
+ /**
+ * Update this sketch with a given unsigned 8-bit integer.
+ * For compatibility with Java implementation.
+ * @param value uint8_t to update the sketch with
+ */
+ template<typename FwdUpdate>
+ inline void update(uint8_t key, FwdUpdate&& value);
+
+ /**
+ * Update this sketch with a given signed 8-bit integer.
+ * For compatibility with Java implementation.
+ * @param value int8_t to update the sketch with
+ */
+ template<typename FwdUpdate>
+ inline void update(int8_t key, FwdUpdate&& value);
+
+ /**
+ * Update this sketch with a given double-precision floating point value.
+ * For compatibility with Java implementation.
+ * @param value double to update the sketch with
+ */
+ template<typename FwdUpdate>
+ inline void update(double key, FwdUpdate&& value);
+
+ /**
+ * Update this sketch with a given floating point value.
+ * For compatibility with Java implementation.
+ * @param value float to update the sketch with
+ */
+ template<typename FwdUpdate>
+ inline void update(float key, FwdUpdate&& value);
+
+ /**
+ * Update this sketch with given data of any type.
+ * This is a "universal" update that covers all cases above,
+ * but may produce different hashes.
+ * Be very careful to hash input values consistently using the same approach
+ * both over time and on different platforms
+ * and while passing sketches between C++ environment and Java environment.
+ * Otherwise two sketches that should represent overlapping sets will be disjoint
+ * For instance, for signed 32-bit values call update(int32_t) method above,
+ * which does widening conversion to int64_t, if compatibility with Java is expected
+ * @param data pointer to the data
+ * @param length of the data in bytes
+ */
+ template<typename FwdUpdate>
+ void update(const void* key, size_t length, FwdUpdate&& value);
+
+ /**
+ * Remove retained entries in excess of the nominal size k (if any)
+ */
+ void trim();
+
+ /**
+ * Converts this sketch to a compact sketch (ordered or unordered).
+ * @param ordered optional flag to specify if ordered sketch should be produced
+ * @return compact sketch
+ */
+ compact_tuple_sketch<Summary, Allocator> compact(bool ordered = true) const;
+
+ virtual iterator begin();
+ virtual iterator end();
+ virtual const_iterator begin() const;
+ virtual const_iterator end() const;
+
+protected:
+ Policy policy_;
+ tuple_map map_;
+
+ // for builder
+ update_tuple_sketch(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta, uint64_t seed, const Policy& policy, const Allocator& allocator);
+
+ virtual void print_specifics(std::basic_ostream<char>& os) const;
+};
+
+// compact sketch
+
+template<
+ typename Summary,
+ typename Allocator = std::allocator<Summary>
+>
+class compact_tuple_sketch: public tuple_sketch<Summary, Allocator> {
+public:
+ using Base = tuple_sketch<Summary, Allocator>;
+ using Entry = typename Base::Entry;
+ using ExtractKey = typename Base::ExtractKey;
+ using iterator = typename Base::iterator;
+ using const_iterator = typename Base::const_iterator;
+ using AllocEntry = typename std::allocator_traits<Allocator>::template rebind_alloc<Entry>;
+ using AllocU64 = typename std::allocator_traits<Allocator>::template rebind_alloc<uint64_t>;
+ using AllocBytes = typename std::allocator_traits<Allocator>::template rebind_alloc<uint8_t>;
+ using vector_bytes = std::vector<uint8_t, AllocBytes>;
+ using comparator = compare_by_key<ExtractKey>;
+
+ static const uint8_t SERIAL_VERSION = 1;
+ static const uint8_t SKETCH_FAMILY = 9;
+ static const uint8_t SKETCH_TYPE = 5;
+ enum flags { IS_BIG_ENDIAN, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED };
+
+ // Instances of this type can be obtained:
+ // - by compacting an update_tuple_sketch
+ // - as a result of a set operation
+ // - by deserializing a previously serialized compact sketch
+
+ compact_tuple_sketch(const Base& other, bool ordered);
+ compact_tuple_sketch(const compact_tuple_sketch&) = default;
+ compact_tuple_sketch(compact_tuple_sketch&&) noexcept;
+ virtual ~compact_tuple_sketch() = default;
+ compact_tuple_sketch& operator=(const compact_tuple_sketch&) = default;
+ compact_tuple_sketch& operator=(compact_tuple_sketch&&) = default;
+
+ compact_tuple_sketch(const theta_sketch_experimental<AllocU64>& other, const Summary& summary, bool ordered = true);
+
+ virtual Allocator get_allocator() const;
+ virtual bool is_empty() const;
+ virtual bool is_ordered() const;
+ virtual uint64_t get_theta64() const;
+ virtual uint32_t get_num_retained() const;
+ virtual uint16_t get_seed_hash() const;
+
+ template<typename SerDe = serde<Summary>>
+ void serialize(std::ostream& os, const SerDe& sd = SerDe()) const;
+
+ template<typename SerDe = serde<Summary>>
+ vector_bytes serialize(unsigned header_size_bytes = 0, const SerDe& sd = SerDe()) const;
+
+ virtual iterator begin();
+ virtual iterator end();
+ virtual const_iterator begin() const;
+ virtual const_iterator end() const;
+
+ /**
+ * This method deserializes a sketch from a given stream.
+ * @param is input stream
+ * @param seed the seed for the hash function that was used to create the sketch
+ * @param instance of a SerDe
+ * @return an instance of a sketch
+ */
+ template<typename SerDe = serde<Summary>>
+ static compact_tuple_sketch deserialize(std::istream& is, uint64_t seed = DEFAULT_SEED,
+ const SerDe& sd = SerDe(), const Allocator& allocator = Allocator());
+
+ /**
+ * This method deserializes a sketch from a given array of bytes.
+ * @param bytes pointer to the array of bytes
+ * @param size the size of the array
+ * @param seed the seed for the hash function that was used to create the sketch
+ * @param instance of a SerDe
+ * @return an instance of the sketch
+ */
+ template<typename SerDe = serde<Summary>>
+ static compact_tuple_sketch deserialize(const void* bytes, size_t size, uint64_t seed = DEFAULT_SEED,
+ const SerDe& sd = SerDe(), const Allocator& allocator = Allocator());
+
+ // for internal use
+ compact_tuple_sketch(bool is_empty, bool is_ordered, uint16_t seed_hash, uint64_t theta, std::vector<Entry, AllocEntry>&& entries);
+
+protected:
+ bool is_empty_;
+ bool is_ordered_;
+ uint16_t seed_hash_;
+ uint64_t theta_;
+ std::vector<Entry, AllocEntry> entries_;
+
+ /**
+ * Computes size needed to serialize summaries in the sketch.
+ * This version is for fixed-size arithmetic types (integral and floating point).
+ * @return size in bytes needed to serialize summaries in this sketch
+ */
+ template<typename SerDe, typename SS = Summary, typename std::enable_if<std::is_arithmetic<SS>::value, int>::type = 0>
+ size_t get_serialized_size_summaries_bytes(const SerDe& sd) const;
+
+ /**
+ * Computes size needed to serialize summaries in the sketch.
+ * This version is for all other types and can be expensive since every item needs to be looked at.
+ * @return size in bytes needed to serialize summaries in this sketch
+ */
+ template<typename SerDe, typename SS = Summary, typename std::enable_if<!std::is_arithmetic<SS>::value, int>::type = 0>
+ size_t get_serialized_size_summaries_bytes(const SerDe& sd) const;
+
+ // for deserialize
+ class deleter_of_summaries {
+ public:
+ deleter_of_summaries(uint32_t num, bool destroy): num(num), destroy(destroy) {}
+ void set_destroy(bool destroy) { this->destroy = destroy; }
+ void operator() (Summary* ptr) const {
+ if (ptr != nullptr) {
+ if (destroy) {
+ for (uint32_t i = 0; i < num; ++i) ptr[i].~Summary();
+ }
+ Allocator().deallocate(ptr, num);
+ }
+ }
+ private:
+ uint32_t num;
+ bool destroy;
+ };
+
+ virtual void print_specifics(std::basic_ostream<char>& os) const;
+
+};
+
+// builder
+
+template<typename Derived, typename Policy, typename Allocator>
+class tuple_base_builder: public theta_base_builder<Derived, Allocator> {
+public:
+ tuple_base_builder(const Policy& policy, const Allocator& allocator);
+
+protected:
+ Policy policy_;
+};
+
+template<typename S, typename U, typename P, typename A>
+class update_tuple_sketch<S, U, P, A>::builder: public tuple_base_builder<builder, P, A> {
+public:
+ /**
+ * Creates and instance of the builder with default parameters.
+ */
+ builder(const P& policy = P(), const A& allocator = A());
+
+ /**
+ * This is to create an instance of the sketch with predefined parameters.
+ * @return an instance of the sketch
+ */
+ update_tuple_sketch<S, U, P, A> build() const;
+};
+
+} /* namespace datasketches */
+
+#include "tuple_sketch_impl.hpp"
+
+#endif
diff --git a/tuple/include/tuple_sketch_impl.hpp b/tuple/include/tuple_sketch_impl.hpp
new file mode 100644
index 0000000..63552d7
--- /dev/null
+++ b/tuple/include/tuple_sketch_impl.hpp
@@ -0,0 +1,587 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <sstream>
+
+#include "binomial_bounds.hpp"
+#include "theta_helpers.hpp"
+
+namespace datasketches {
+
+template<typename S, typename A>
+bool tuple_sketch<S, A>::is_estimation_mode() const {
+ return get_theta64() < theta_constants::MAX_THETA && !is_empty();
+}
+
+template<typename S, typename A>
+double tuple_sketch<S, A>::get_theta() const {
+ return static_cast<double>(get_theta64()) / theta_constants::MAX_THETA;
+}
+
+template<typename S, typename A>
+double tuple_sketch<S, A>::get_estimate() const {
+ return get_num_retained() / get_theta();
+}
+
+template<typename S, typename A>
+double tuple_sketch<S, A>::get_lower_bound(uint8_t num_std_devs) const {
+ if (!is_estimation_mode()) return get_num_retained();
+ return binomial_bounds::get_lower_bound(get_num_retained(), get_theta(), num_std_devs);
+}
+
+template<typename S, typename A>
+double tuple_sketch<S, A>::get_upper_bound(uint8_t num_std_devs) const {
+ if (!is_estimation_mode()) return get_num_retained();
+ return binomial_bounds::get_upper_bound(get_num_retained(), get_theta(), num_std_devs);
+}
+
+template<typename S, typename A>
+string<A> tuple_sketch<S, A>::to_string(bool detail) const {
+ std::basic_ostringstream<char, std::char_traits<char>, AllocChar<A>> os;
+ os << "### Tuple sketch summary:" << std::endl;
+ os << " num retained entries : " << get_num_retained() << std::endl;
+ os << " seed hash : " << get_seed_hash() << std::endl;
+ os << " empty? : " << (is_empty() ? "true" : "false") << std::endl;
+ os << " ordered? : " << (is_ordered() ? "true" : "false") << std::endl;
+ os << " estimation mode? : " << (is_estimation_mode() ? "true" : "false") << std::endl;
+ os << " theta (fraction) : " << get_theta() << std::endl;
+ os << " theta (raw 64-bit) : " << get_theta64() << std::endl;
+ os << " estimate : " << this->get_estimate() << std::endl;
+ os << " lower bound 95% conf : " << this->get_lower_bound(2) << std::endl;
+ os << " upper bound 95% conf : " << this->get_upper_bound(2) << std::endl;
+ print_specifics(os);
+ os << "### End sketch summary" << std::endl;
+ if (detail) {
+ os << "### Retained entries" << std::endl;
+ for (const auto& it: *this) {
+ os << it.first << ": " << it.second << std::endl;
+ }
+ os << "### End retained entries" << std::endl;
+ }
+ return os.str();
+}
+
+// update sketch
+
+template<typename S, typename U, typename P, typename A>
+update_tuple_sketch<S, U, P, A>::update_tuple_sketch(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta, uint64_t seed, const P& policy, const A& allocator):
+policy_(policy),
+map_(lg_cur_size, lg_nom_size, rf, theta, seed, allocator)
+{}
+
+template<typename S, typename U, typename P, typename A>
+A update_tuple_sketch<S, U, P, A>::get_allocator() const {
+ return map_.allocator_;
+}
+
+template<typename S, typename U, typename P, typename A>
+bool update_tuple_sketch<S, U, P, A>::is_empty() const {
+ return map_.is_empty_;
+}
+
+template<typename S, typename U, typename P, typename A>
+bool update_tuple_sketch<S, U, P, A>::is_ordered() const {
+ return false;
+}
+
+template<typename S, typename U, typename P, typename A>
+uint64_t update_tuple_sketch<S, U, P, A>::get_theta64() const {
+ return map_.theta_;
+}
+
+template<typename S, typename U, typename P, typename A>
+uint32_t update_tuple_sketch<S, U, P, A>::get_num_retained() const {
+ return map_.num_entries_;
+}
+
+template<typename S, typename U, typename P, typename A>
+uint16_t update_tuple_sketch<S, U, P, A>::get_seed_hash() const {
+ return compute_seed_hash(map_.seed_);
+}
+
+template<typename S, typename U, typename P, typename A>
+uint8_t update_tuple_sketch<S, U, P, A>::get_lg_k() const {
+ return map_.lg_nom_size_;
+}
+
+template<typename S, typename U, typename P, typename A>
+auto update_tuple_sketch<S, U, P, A>::get_rf() const -> resize_factor {
+ return map_.rf_;
+}
+
+template<typename S, typename U, typename P, typename A>
+template<typename UU>
+void update_tuple_sketch<S, U, P, A>::update(uint64_t key, UU&& value) {
+ update(&key, sizeof(key), std::forward<UU>(value));
+}
+
+template<typename S, typename U, typename P, typename A>
+template<typename UU>
+void update_tuple_sketch<S, U, P, A>::update(int64_t key, UU&& value) {
+ update(&key, sizeof(key), std::forward<UU>(value));
+}
+
+template<typename S, typename U, typename P, typename A>
+template<typename UU>
+void update_tuple_sketch<S, U, P, A>::update(uint32_t key, UU&& value) {
+ update(static_cast<int32_t>(key), std::forward<UU>(value));
+}
+
+template<typename S, typename U, typename P, typename A>
+template<typename UU>
+void update_tuple_sketch<S, U, P, A>::update(int32_t key, UU&& value) {
+ update(static_cast<int64_t>(key), std::forward<UU>(value));
+}
+
+template<typename S, typename U, typename P, typename A>
+template<typename UU>
+void update_tuple_sketch<S, U, P, A>::update(uint16_t key, UU&& value) {
+ update(static_cast<int16_t>(key), std::forward<UU>(value));
+}
+
+template<typename S, typename U, typename P, typename A>
+template<typename UU>
+void update_tuple_sketch<S, U, P, A>::update(int16_t key, UU&& value) {
+ update(static_cast<int64_t>(key), std::forward<UU>(value));
+}
+
+template<typename S, typename U, typename P, typename A>
+template<typename UU>
+void update_tuple_sketch<S, U, P, A>::update(uint8_t key, UU&& value) {
+ update(static_cast<int8_t>(key), std::forward<UU>(value));
+}
+
+template<typename S, typename U, typename P, typename A>
+template<typename UU>
+void update_tuple_sketch<S, U, P, A>::update(int8_t key, UU&& value) {
+ update(static_cast<int64_t>(key), std::forward<UU>(value));
+}
+
+template<typename S, typename U, typename P, typename A>
+template<typename UU>
+void update_tuple_sketch<S, U, P, A>::update(const std::string& key, UU&& value) {
+ if (key.empty()) return;
+ update(key.c_str(), key.length(), std::forward<UU>(value));
+}
+
+template<typename S, typename U, typename P, typename A>
+template<typename UU>
+void update_tuple_sketch<S, U, P, A>::update(double key, UU&& value) {
+ update(canonical_double(key), std::forward<UU>(value));
+}
+
+template<typename S, typename U, typename P, typename A>
+template<typename UU>
+void update_tuple_sketch<S, U, P, A>::update(float key, UU&& value) {
+ update(static_cast<double>(key), std::forward<UU>(value));
+}
+
+template<typename S, typename U, typename P, typename A>
+template<typename UU>
+void update_tuple_sketch<S, U, P, A>::update(const void* key, size_t length, UU&& value) {
+ const uint64_t hash = map_.hash_and_screen(key, length);
+ if (hash == 0) return;
+ auto result = map_.find(hash);
+ if (!result.second) {
+ S summary = policy_.create();
+ policy_.update(summary, std::forward<UU>(value));
+ map_.insert(result.first, Entry(hash, std::move(summary)));
+ } else {
+ policy_.update((*result.first).second, std::forward<UU>(value));
+ }
+}
+
+template<typename S, typename U, typename P, typename A>
+void update_tuple_sketch<S, U, P, A>::trim() {
+ map_.trim();
+}
+
+template<typename S, typename U, typename P, typename A>
+auto update_tuple_sketch<S, U, P, A>::begin() -> iterator {
+ return iterator(map_.entries_, 1 << map_.lg_cur_size_, 0);
+}
+
+template<typename S, typename U, typename P, typename A>
+auto update_tuple_sketch<S, U, P, A>::end() -> iterator {
+ return iterator(nullptr, 0, 1 << map_.lg_cur_size_);
+}
+
+template<typename S, typename U, typename P, typename A>
+auto update_tuple_sketch<S, U, P, A>::begin() const -> const_iterator {
+ return const_iterator(map_.entries_, 1 << map_.lg_cur_size_, 0);
+}
+
+template<typename S, typename U, typename P, typename A>
+auto update_tuple_sketch<S, U, P, A>::end() const -> const_iterator {
+ return const_iterator(nullptr, 0, 1 << map_.lg_cur_size_);
+}
+
+template<typename S, typename U, typename P, typename A>
+compact_tuple_sketch<S, A> update_tuple_sketch<S, U, P, A>::compact(bool ordered) const {
+ return compact_tuple_sketch<S, A>(*this, ordered);
+}
+
+template<typename S, typename U, typename P, typename A>
+void update_tuple_sketch<S, U, P, A>::print_specifics(std::basic_ostream<char>& os) const {
+ os << " lg nominal size : " << (int) map_.lg_nom_size_ << std::endl;
+ os << " lg current size : " << (int) map_.lg_cur_size_ << std::endl;
+ os << " resize factor : " << (1 << map_.rf_) << std::endl;
+}
+
+// compact sketch
+
+template<typename S, typename A>
+compact_tuple_sketch<S, A>::compact_tuple_sketch(bool is_empty, bool is_ordered, uint16_t seed_hash, uint64_t theta,
+ std::vector<Entry, AllocEntry>&& entries):
+is_empty_(is_empty),
+is_ordered_(is_ordered),
+seed_hash_(seed_hash),
+theta_(theta),
+entries_(std::move(entries))
+{}
+
+template<typename S, typename A>
+compact_tuple_sketch<S, A>::compact_tuple_sketch(const Base& other, bool ordered):
+is_empty_(other.is_empty()),
+is_ordered_(other.is_ordered() || ordered),
+seed_hash_(other.get_seed_hash()),
+theta_(other.get_theta64()),
+entries_(other.get_allocator())
+{
+ entries_.reserve(other.get_num_retained());
+ std::copy(other.begin(), other.end(), std::back_inserter(entries_));
+ if (ordered && !other.is_ordered()) std::sort(entries_.begin(), entries_.end(), comparator());
+}
+
+template<typename S, typename A>
+compact_tuple_sketch<S, A>::compact_tuple_sketch(compact_tuple_sketch&& other) noexcept:
+is_empty_(other.is_empty()),
+is_ordered_(other.is_ordered()),
+seed_hash_(other.get_seed_hash()),
+theta_(other.get_theta64()),
+entries_(std::move(other.entries_))
+{}
+
+template<typename S, typename A>
+compact_tuple_sketch<S, A>::compact_tuple_sketch(const theta_sketch_experimental<AllocU64>& other, const S& summary, bool ordered):
+is_empty_(other.is_empty()),
+is_ordered_(other.is_ordered() || ordered),
+seed_hash_(other.get_seed_hash()),
+theta_(other.get_theta64()),
+entries_(other.get_allocator())
+{
+ entries_.reserve(other.get_num_retained());
+ for (uint64_t hash: other) {
+ entries_.push_back(Entry(hash, summary));
+ }
+ if (ordered && !other.is_ordered()) std::sort(entries_.begin(), entries_.end(), comparator());
+}
+
+template<typename S, typename A>
+A compact_tuple_sketch<S, A>::get_allocator() const {
+ return entries_.get_allocator();
+}
+
+template<typename S, typename A>
+bool compact_tuple_sketch<S, A>::is_empty() const {
+ return is_empty_;
+}
+
+template<typename S, typename A>
+bool compact_tuple_sketch<S, A>::is_ordered() const {
+ return is_ordered_;
+}
+
+template<typename S, typename A>
+uint64_t compact_tuple_sketch<S, A>::get_theta64() const {
+ return theta_;
+}
+
+template<typename S, typename A>
+uint32_t compact_tuple_sketch<S, A>::get_num_retained() const {
+ return entries_.size();
+}
+
+template<typename S, typename A>
+uint16_t compact_tuple_sketch<S, A>::get_seed_hash() const {
+ return seed_hash_;
+}
+
+// implementation for fixed-size arithmetic types (integral and floating point)
+template<typename S, typename A>
+template<typename SD, typename SS, typename std::enable_if<std::is_arithmetic<SS>::value, int>::type>
+size_t compact_tuple_sketch<S, A>::get_serialized_size_summaries_bytes(const SD& sd) const {
+ unused(sd);
+ return entries_.size() * sizeof(SS);
+}
+
+// implementation for all other types (non-arithmetic)
+template<typename S, typename A>
+template<typename SD, typename SS, typename std::enable_if<!std::is_arithmetic<SS>::value, int>::type>
+size_t compact_tuple_sketch<S, A>::get_serialized_size_summaries_bytes(const SD& sd) const {
+ size_t size = 0;
+ for (const auto& it: entries_) {
+ size += sd.size_of_item(it.second);
+ }
+ return size;
+}
+
+template<typename S, typename A>
+template<typename SerDe>
+void compact_tuple_sketch<S, A>::serialize(std::ostream& os, const SerDe& sd) const {
+ const bool is_single_item = entries_.size() == 1 && !this->is_estimation_mode();
+ const uint8_t preamble_longs = this->is_empty() || is_single_item ? 1 : this->is_estimation_mode() ? 3 : 2;
+ os.write(reinterpret_cast<const char*>(&preamble_longs), sizeof(preamble_longs));
+ const uint8_t serial_version = SERIAL_VERSION;
+ os.write(reinterpret_cast<const char*>(&serial_version), sizeof(serial_version));
+ const uint8_t family = SKETCH_FAMILY;
+ os.write(reinterpret_cast<const char*>(&family), sizeof(family));
+ const uint8_t type = SKETCH_TYPE;
+ os.write(reinterpret_cast<const char*>(&type), sizeof(type));
+ const uint8_t unused8 = 0;
+ os.write(reinterpret_cast<const char*>(&unused8), sizeof(unused8));
+ const uint8_t flags_byte(
+ (1 << flags::IS_COMPACT) |
+ (1 << flags::IS_READ_ONLY) |
+ (this->is_empty() ? 1 << flags::IS_EMPTY : 0) |
+ (this->is_ordered() ? 1 << flags::IS_ORDERED : 0)
+ );
+ os.write(reinterpret_cast<const char*>(&flags_byte), sizeof(flags_byte));
+ const uint16_t seed_hash = get_seed_hash();
+ os.write(reinterpret_cast<const char*>(&seed_hash), sizeof(seed_hash));
+ if (!this->is_empty()) {
+ if (!is_single_item) {
+ const uint32_t num_entries = entries_.size();
+ os.write(reinterpret_cast<const char*>(&num_entries), sizeof(num_entries));
+ const uint32_t unused32 = 0;
+ os.write(reinterpret_cast<const char*>(&unused32), sizeof(unused32));
+ if (this->is_estimation_mode()) {
+ os.write(reinterpret_cast<const char*>(&(this->theta_)), sizeof(uint64_t));
+ }
+ }
+ for (const auto& it: entries_) {
+ os.write(reinterpret_cast<const char*>(&it.first), sizeof(uint64_t));
+ sd.serialize(os, &it.second, 1);
+ }
+ }
+}
+
+template<typename S, typename A>
+template<typename SerDe>
+auto compact_tuple_sketch<S, A>::serialize(unsigned header_size_bytes, const SerDe& sd) const -> vector_bytes {
+ const bool is_single_item = entries_.size() == 1 && !this->is_estimation_mode();
+ const uint8_t preamble_longs = this->is_empty() || is_single_item ? 1 : this->is_estimation_mode() ? 3 : 2;
+ const size_t size = header_size_bytes + sizeof(uint64_t) * preamble_longs
+ + sizeof(uint64_t) * entries_.size() + get_serialized_size_summaries_bytes(sd);
+ vector_bytes bytes(size, 0, entries_.get_allocator());
+ uint8_t* ptr = bytes.data() + header_size_bytes;
+ const uint8_t* end_ptr = ptr + size;
+
+ ptr += copy_to_mem(&preamble_longs, ptr, sizeof(preamble_longs));
+ const uint8_t serial_version = SERIAL_VERSION;
+ ptr += copy_to_mem(&serial_version, ptr, sizeof(serial_version));
+ const uint8_t family = SKETCH_FAMILY;
+ ptr += copy_to_mem(&family, ptr, sizeof(family));
+ const uint8_t type = SKETCH_TYPE;
+ ptr += copy_to_mem(&type, ptr, sizeof(type));
+ const uint8_t unused8 = 0;
+ ptr += copy_to_mem(&unused8, ptr, sizeof(unused8));
+ const uint8_t flags_byte(
+ (1 << flags::IS_COMPACT) |
+ (1 << flags::IS_READ_ONLY) |
+ (this->is_empty() ? 1 << flags::IS_EMPTY : 0) |
+ (this->is_ordered() ? 1 << flags::IS_ORDERED : 0)
+ );
+ ptr += copy_to_mem(&flags_byte, ptr, sizeof(flags_byte));
+ const uint16_t seed_hash = get_seed_hash();
+ ptr += copy_to_mem(&seed_hash, ptr, sizeof(seed_hash));
+ if (!this->is_empty()) {
+ if (!is_single_item) {
+ const uint32_t num_entries = entries_.size();
+ ptr += copy_to_mem(&num_entries, ptr, sizeof(num_entries));
+ const uint32_t unused32 = 0;
+ ptr += copy_to_mem(&unused32, ptr, sizeof(unused32));
+ if (this->is_estimation_mode()) {
+ ptr += copy_to_mem(&theta_, ptr, sizeof(uint64_t));
+ }
+ }
+ for (const auto& it: entries_) {
+ ptr += copy_to_mem(&it.first, ptr, sizeof(uint64_t));
+ ptr += sd.serialize(ptr, end_ptr - ptr, &it.second, 1);
+ }
+ }
+ return bytes;
+}
+
+template<typename S, typename A>
+template<typename SerDe>
+compact_tuple_sketch<S, A> compact_tuple_sketch<S, A>::deserialize(std::istream& is, uint64_t seed, const SerDe& sd, const A& allocator) {
+ uint8_t preamble_longs;
+ is.read(reinterpret_cast<char*>(&preamble_longs), sizeof(preamble_longs));
+ uint8_t serial_version;
+ is.read(reinterpret_cast<char*>(&serial_version), sizeof(serial_version));
+ uint8_t family;
+ is.read(reinterpret_cast<char*>(&family), sizeof(family));
+ uint8_t type;
+ is.read(reinterpret_cast<char*>(&type), sizeof(type));
+ uint8_t unused8;
+ is.read(reinterpret_cast<char*>(&unused8), sizeof(unused8));
+ uint8_t flags_byte;
+ is.read(reinterpret_cast<char*>(&flags_byte), sizeof(flags_byte));
+ uint16_t seed_hash;
+ is.read(reinterpret_cast<char*>(&seed_hash), sizeof(seed_hash));
+ checker<true>::check_serial_version(serial_version, SERIAL_VERSION);
+ checker<true>::check_sketch_family(family, SKETCH_FAMILY);
+ checker<true>::check_sketch_type(type, SKETCH_TYPE);
+ const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
+ if (!is_empty) checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+
+ uint64_t theta = theta_constants::MAX_THETA;
+ uint32_t num_entries = 0;
+ if (!is_empty) {
+ if (preamble_longs == 1) {
+ num_entries = 1;
+ } else {
+ is.read(reinterpret_cast<char*>(&num_entries), sizeof(num_entries));
+ uint32_t unused32;
+ is.read(reinterpret_cast<char*>(&unused32), sizeof(unused32));
+ if (preamble_longs > 2) {
+ is.read(reinterpret_cast<char*>(&theta), sizeof(theta));
+ }
+ }
+ }
+ A alloc(allocator);
+ std::vector<Entry, AllocEntry> entries(alloc);
+ if (!is_empty) {
+ entries.reserve(num_entries);
+ std::unique_ptr<S, deleter_of_summaries> summary(alloc.allocate(1), deleter_of_summaries(1, false));
+ for (size_t i = 0; i < num_entries; ++i) {
+ uint64_t key;
+ is.read(reinterpret_cast<char*>(&key), sizeof(uint64_t));
+ sd.deserialize(is, summary.get(), 1);
+ entries.push_back(Entry(key, std::move(*summary)));
+ (*summary).~S();
+ }
+ }
+ if (!is.good()) throw std::runtime_error("error reading from std::istream");
+ const bool is_ordered = flags_byte & (1 << flags::IS_ORDERED);
+ return compact_tuple_sketch(is_empty, is_ordered, seed_hash, theta, std::move(entries));
+}
+
+template<typename S, typename A>
+template<typename SerDe>
+compact_tuple_sketch<S, A> compact_tuple_sketch<S, A>::deserialize(const void* bytes, size_t size, uint64_t seed, const SerDe& sd, const A& allocator) {
+ ensure_minimum_memory(size, 8);
+ const char* ptr = static_cast<const char*>(bytes);
+ const char* base = ptr;
+ uint8_t preamble_longs;
+ ptr += copy_from_mem(ptr, &preamble_longs, sizeof(preamble_longs));
+ uint8_t serial_version;
+ ptr += copy_from_mem(ptr, &serial_version, sizeof(serial_version));
+ uint8_t family;
+ ptr += copy_from_mem(ptr, &family, sizeof(family));
+ uint8_t type;
+ ptr += copy_from_mem(ptr, &type, sizeof(type));
+ uint8_t unused8;
+ ptr += copy_from_mem(ptr, &unused8, sizeof(unused8));
+ uint8_t flags_byte;
+ ptr += copy_from_mem(ptr, &flags_byte, sizeof(flags_byte));
+ uint16_t seed_hash;
+ ptr += copy_from_mem(ptr, &seed_hash, sizeof(seed_hash));
+ checker<true>::check_serial_version(serial_version, SERIAL_VERSION);
+ checker<true>::check_sketch_family(family, SKETCH_FAMILY);
+ checker<true>::check_sketch_type(type, SKETCH_TYPE);
+ const bool is_empty = flags_byte & (1 << flags::IS_EMPTY);
+ if (!is_empty) checker<true>::check_seed_hash(seed_hash, compute_seed_hash(seed));
+
+ uint64_t theta = theta_constants::MAX_THETA;
+ uint32_t num_entries = 0;
+
+ if (!is_empty) {
+ if (preamble_longs == 1) {
+ num_entries = 1;
+ } else {
+ ensure_minimum_memory(size, 8); // read the first prelong before this method
+ ptr += copy_from_mem(ptr, &num_entries, sizeof(num_entries));
+ uint32_t unused32;
+ ptr += copy_from_mem(ptr, &unused32, sizeof(unused32));
+ if (preamble_longs > 2) {
+ ensure_minimum_memory(size, (preamble_longs - 1) << 3);
+ ptr += copy_from_mem(ptr, &theta, sizeof(theta));
+ }
+ }
+ }
+ const size_t keys_size_bytes = sizeof(uint64_t) * num_entries;
+ ensure_minimum_memory(size, ptr - base + keys_size_bytes);
+ A alloc(allocator);
+ std::vector<Entry, AllocEntry> entries(alloc);
+ if (!is_empty) {
+ entries.reserve(num_entries);
+ std::unique_ptr<S, deleter_of_summaries> summary(alloc.allocate(1), deleter_of_summaries(1, false));
+ for (size_t i = 0; i < num_entries; ++i) {
+ uint64_t key;
+ ptr += copy_from_mem(ptr, &key, sizeof(key));
+ ptr += sd.deserialize(ptr, base + size - ptr, summary.get(), 1);
+ entries.push_back(Entry(key, std::move(*summary)));
+ (*summary).~S();
+ }
+ }
+ const bool is_ordered = flags_byte & (1 << flags::IS_ORDERED);
+ return compact_tuple_sketch(is_empty, is_ordered, seed_hash, theta, std::move(entries));
+}
+
+template<typename S, typename A>
+auto compact_tuple_sketch<S, A>::begin() -> iterator {
+ return iterator(entries_.data(), entries_.size(), 0);
+}
+
+template<typename S, typename A>
+auto compact_tuple_sketch<S, A>::end() -> iterator {
+ return iterator(nullptr, 0, entries_.size());
+}
+
+template<typename S, typename A>
+auto compact_tuple_sketch<S, A>::begin() const -> const_iterator {
+ return const_iterator(entries_.data(), entries_.size(), 0);
+}
+
+template<typename S, typename A>
+auto compact_tuple_sketch<S, A>::end() const -> const_iterator {
+ return const_iterator(nullptr, 0, entries_.size());
+}
+
+template<typename S, typename A>
+void compact_tuple_sketch<S, A>::print_specifics(std::basic_ostream<char>&) const {}
+
+// builder
+
+template<typename D, typename P, typename A>
+tuple_base_builder<D, P, A>::tuple_base_builder(const P& policy, const A& allocator):
+theta_base_builder<D, A>(allocator), policy_(policy) {}
+
+template<typename S, typename U, typename P, typename A>
+update_tuple_sketch<S, U, P, A>::builder::builder(const P& policy, const A& allocator):
+tuple_base_builder<builder, P, A>(policy, allocator) {}
+
+template<typename S, typename U, typename P, typename A>
+auto update_tuple_sketch<S, U, P, A>::builder::build() const -> update_tuple_sketch {
+ return update_tuple_sketch(this->starting_lg_size(), this->lg_k_, this->rf_, this->starting_theta(), this->seed_, this->policy_, this->allocator_);
+}
+
+} /* namespace datasketches */
diff --git a/tuple/include/tuple_union.hpp b/tuple/include/tuple_union.hpp
new file mode 100644
index 0000000..d9eff26
--- /dev/null
+++ b/tuple/include/tuple_union.hpp
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef TUPLE_UNION_HPP_
+#define TUPLE_UNION_HPP_
+
+#include "tuple_sketch.hpp"
+#include "theta_union_base.hpp"
+
+namespace datasketches {
+
+// for types with defined + operation
+template<typename Summary>
+struct default_union_policy {
+ void operator()(Summary& summary, const Summary& other) const {
+ summary += other;
+ }
+};
+
+template<
+ typename Summary,
+ typename Policy = default_union_policy<Summary>,
+ typename Allocator = std::allocator<Summary>
+>
+class tuple_union {
+public:
+ using Entry = std::pair<uint64_t, Summary>;
+ using ExtractKey = pair_extract_key<uint64_t, Summary>;
+ using Sketch = tuple_sketch<Summary, Allocator>;
+ using CompactSketch = compact_tuple_sketch<Summary, Allocator>;
+ using AllocEntry = typename std::allocator_traits<Allocator>::template rebind_alloc<Entry>;
+ using resize_factor = theta_constants::resize_factor;
+
+ // reformulate the external policy that operates on Summary
+ // in terms of operations on Entry
+ struct internal_policy {
+ internal_policy(const Policy& policy): policy_(policy) {}
+ void operator()(Entry& internal_entry, const Entry& incoming_entry) const {
+ policy_(internal_entry.second, incoming_entry.second);
+ }
+ void operator()(Entry& internal_entry, Entry&& incoming_entry) const {
+ policy_(internal_entry.second, std::move(incoming_entry.second));
+ }
+ const Policy& get_policy() const { return policy_; }
+ Policy policy_;
+ };
+
+ using State = theta_union_base<Entry, ExtractKey, internal_policy, Sketch, CompactSketch, AllocEntry>;
+
+ // No constructor here. Use builder instead.
+ class builder;
+
+ /**
+ * This method is to update the union with a given sketch
+ * @param sketch to update the union with
+ */
+ template<typename FwdSketch>
+ void update(FwdSketch&& sketch);
+
+ /**
+ * This method produces a copy of the current state of the union as a compact sketch.
+ * @param ordered optional flag to specify if ordered sketch should be produced
+ * @return the result of the union
+ */
+ CompactSketch get_result(bool ordered = true) const;
+
+protected:
+ State state_;
+
+ // for builder
+ tuple_union(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta, uint64_t seed, const Policy& policy, const Allocator& allocator);
+};
+
+template<typename S, typename P, typename A>
+class tuple_union<S, P, A>::builder: public tuple_base_builder<builder, P, A> {
+public:
+ /**
+ * Creates and instance of the builder with default parameters.
+ */
+ builder(const P& policy = P(), const A& allocator = A());
+
+ /**
+ * This is to create an instance of the union with predefined parameters.
+ * @return an instance of the union
+ */
+ tuple_union build() const;
+};
+
+} /* namespace datasketches */
+
+#include "tuple_union_impl.hpp"
+
+#endif
diff --git a/tuple/include/tuple_union_impl.hpp b/tuple/include/tuple_union_impl.hpp
new file mode 100644
index 0000000..4b3b0a5
--- /dev/null
+++ b/tuple/include/tuple_union_impl.hpp
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+namespace datasketches {
+
+template<typename S, typename P, typename A>
+tuple_union<S, P, A>::tuple_union(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, uint64_t theta, uint64_t seed, const P& policy, const A& allocator):
+state_(lg_cur_size, lg_nom_size, rf, theta, seed, internal_policy(policy), allocator)
+{}
+
+template<typename S, typename P, typename A>
+template<typename SS>
+void tuple_union<S, P, A>::update(SS&& sketch) {
+ state_.update(std::forward<SS>(sketch));
+}
+
+template<typename S, typename P, typename A>
+auto tuple_union<S, P, A>::get_result(bool ordered) const -> CompactSketch {
+ return state_.get_result(ordered);
+}
+
+template<typename S, typename P, typename A>
+tuple_union<S, P, A>::builder::builder(const P& policy, const A& allocator):
+tuple_base_builder<builder, P, A>(policy, allocator) {}
+
+template<typename S, typename P, typename A>
+auto tuple_union<S, P, A>::builder::build() const -> tuple_union {
+ return tuple_union(this->starting_lg_size(), this->lg_k_, this->rf_, this->starting_theta(), this->seed_, this->policy_, this->allocator_);
+}
+
+} /* namespace datasketches */
diff --git a/tuple/test/CMakeLists.txt b/tuple/test/CMakeLists.txt
new file mode 100644
index 0000000..cf87aaa
--- /dev/null
+++ b/tuple/test/CMakeLists.txt
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+add_executable(tuple_test)
+
+target_link_libraries(tuple_test tuple common_test)
+
+set_target_properties(tuple_test PROPERTIES
+ CXX_STANDARD 11
+ CXX_STANDARD_REQUIRED YES
+)
+
+file(TO_CMAKE_PATH "${CMAKE_CURRENT_SOURCE_DIR}" THETA_TEST_BINARY_PATH)
+string(APPEND THETA_TEST_BINARY_PATH "/")
+target_compile_definitions(tuple_test
+ PRIVATE
+ TEST_BINARY_INPUT_PATH="${THETA_TEST_BINARY_PATH}"
+)
+
+add_test(
+ NAME tuple_test
+ COMMAND tuple_test
+)
+
+target_sources(tuple_test
+ PRIVATE
+ tuple_sketch_test.cpp
+ tuple_sketch_allocation_test.cpp
+ tuple_union_test.cpp
+ tuple_intersection_test.cpp
+ tuple_a_not_b_test.cpp
+ array_of_doubles_sketch_test.cpp
+ theta_sketch_experimental_test.cpp
+ theta_union_experimental_test.cpp
+ theta_intersection_experimental_test.cpp
+ theta_a_not_b_experimental_test.cpp
+ theta_jaccard_similarity_test.cpp
+ tuple_jaccard_similarity_test.cpp
+)
diff --git a/tuple/test/aod_1_compact_empty_from_java.sk b/tuple/test/aod_1_compact_empty_from_java.sk
new file mode 100644
index 0000000..8d2583d
--- /dev/null
+++ b/tuple/test/aod_1_compact_empty_from_java.sk
@@ -0,0 +1 @@
+ Ì“ÿÿÿÿÿÿÿ
\ No newline at end of file
diff --git a/tuple/test/aod_1_compact_estimation_from_java.sk b/tuple/test/aod_1_compact_estimation_from_java.sk
new file mode 100644
index 0000000..d086489
--- /dev/null
+++ b/tuple/test/aod_1_compact_estimation_from_java.sk
Binary files differ
diff --git a/tuple/test/aod_1_compact_non_empty_no_entries_from_java.sk b/tuple/test/aod_1_compact_non_empty_no_entries_from_java.sk
new file mode 100644
index 0000000..f67106d
--- /dev/null
+++ b/tuple/test/aod_1_compact_non_empty_no_entries_from_java.sk
Binary files differ
diff --git a/tuple/test/aod_2_compact_exact_from_java.sk b/tuple/test/aod_2_compact_exact_from_java.sk
new file mode 100644
index 0000000..a14fd08
--- /dev/null
+++ b/tuple/test/aod_2_compact_exact_from_java.sk
Binary files differ
diff --git a/tuple/test/aod_3_compact_empty_from_java.sk b/tuple/test/aod_3_compact_empty_from_java.sk
new file mode 100644
index 0000000..1579d9b
--- /dev/null
+++ b/tuple/test/aod_3_compact_empty_from_java.sk
@@ -0,0 +1 @@
+ Ì“ÿÿÿÿÿÿÿ
\ No newline at end of file
diff --git a/tuple/test/array_of_doubles_sketch_test.cpp b/tuple/test/array_of_doubles_sketch_test.cpp
new file mode 100644
index 0000000..fa5fc92
--- /dev/null
+++ b/tuple/test/array_of_doubles_sketch_test.cpp
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <array>
+
+#include <catch.hpp>
+#include <array_of_doubles_sketch.hpp>
+#include <array_of_doubles_union.hpp>
+#include <array_of_doubles_intersection.hpp>
+
+namespace datasketches {
+
+#ifdef TEST_BINARY_INPUT_PATH
+const std::string inputPath = TEST_BINARY_INPUT_PATH;
+#else
+const std::string inputPath = "test/";
+#endif
+
+TEST_CASE("aod sketch: serialization compatibility with java - empty", "[tuple_sketch]") {
+ auto update_sketch = update_array_of_doubles_sketch::builder().build();
+ REQUIRE(update_sketch.is_empty());
+ REQUIRE(update_sketch.get_num_retained() == 0);
+ auto compact_sketch = update_sketch.compact();
+
+ // read binary sketch from Java
+ std::ifstream is;
+ is.exceptions(std::ios::failbit | std::ios::badbit);
+ is.open(inputPath + "aod_1_compact_empty_from_java.sk", std::ios::binary);
+ auto compact_sketch_from_java = compact_array_of_doubles_sketch::deserialize(is);
+ REQUIRE(compact_sketch.get_num_retained() == compact_sketch_from_java.get_num_retained());
+ REQUIRE(compact_sketch.get_theta() == Approx(compact_sketch_from_java.get_theta()).margin(1e-10));
+ REQUIRE(compact_sketch.get_estimate() == Approx(compact_sketch_from_java.get_estimate()).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(1) == Approx(compact_sketch_from_java.get_lower_bound(1)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(1) == Approx(compact_sketch_from_java.get_upper_bound(1)).margin(1e-10));
+}
+
+TEST_CASE("aod sketch: serialization compatibility with java - empty configured for three values", "[tuple_sketch]") {
+ auto update_sketch = update_array_of_doubles_sketch::builder(3).build();
+ REQUIRE(update_sketch.is_empty());
+ REQUIRE(update_sketch.get_num_retained() == 0);
+ REQUIRE(update_sketch.get_num_values() == 3);
+ auto compact_sketch = update_sketch.compact();
+
+ // read binary sketch from Java
+ std::ifstream is;
+ is.exceptions(std::ios::failbit | std::ios::badbit);
+ is.open(inputPath + "aod_3_compact_empty_from_java.sk", std::ios::binary);
+ auto compact_sketch_from_java = compact_array_of_doubles_sketch::deserialize(is);
+ REQUIRE(compact_sketch.get_num_values() == compact_sketch_from_java.get_num_values());
+ REQUIRE(compact_sketch.get_num_retained() == compact_sketch_from_java.get_num_retained());
+ REQUIRE(compact_sketch.get_theta() == Approx(compact_sketch_from_java.get_theta()).margin(1e-10));
+ REQUIRE(compact_sketch.get_estimate() == Approx(compact_sketch_from_java.get_estimate()).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(1) == Approx(compact_sketch_from_java.get_lower_bound(1)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(1) == Approx(compact_sketch_from_java.get_upper_bound(1)).margin(1e-10));
+}
+
+TEST_CASE("aod sketch: serialization compatibility with java - non-empty no entries", "[tuple_sketch]") {
+ auto update_sketch = update_array_of_doubles_sketch::builder().set_p(0.01).build();
+ std::vector<double> a = {1};
+ update_sketch.update(1, a);
+ REQUIRE_FALSE(update_sketch.is_empty());
+ REQUIRE(update_sketch.get_num_retained() == 0);
+ auto compact_sketch = update_sketch.compact();
+
+ // read binary sketch from Java
+ std::ifstream is;
+ is.exceptions(std::ios::failbit | std::ios::badbit);
+ is.open(inputPath + "aod_1_compact_non_empty_no_entries_from_java.sk", std::ios::binary);
+ auto compact_sketch_from_java = compact_array_of_doubles_sketch::deserialize(is);
+ REQUIRE(compact_sketch.get_num_retained() == compact_sketch_from_java.get_num_retained());
+ REQUIRE(compact_sketch.get_theta() == Approx(compact_sketch_from_java.get_theta()).margin(1e-10));
+ REQUIRE(compact_sketch.get_estimate() == Approx(compact_sketch_from_java.get_estimate()).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(1) == Approx(compact_sketch_from_java.get_lower_bound(1)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(1) == Approx(compact_sketch_from_java.get_upper_bound(1)).margin(1e-10));
+}
+
+TEST_CASE("aod sketch: serialization compatibility with java - estimation mode", "[tuple_sketch]") {
+ auto update_sketch = update_array_of_doubles_sketch::builder().build();
+ std::vector<double> a = {1};
+ for (int i = 0; i < 8192; ++i) update_sketch.update(i, a);
+ auto compact_sketch = update_sketch.compact();
+
+ // read binary sketch from Java
+ std::ifstream is;
+ is.exceptions(std::ios::failbit | std::ios::badbit);
+ is.open(inputPath + "aod_1_compact_estimation_from_java.sk", std::ios::binary);
+ auto compact_sketch_from_java = compact_array_of_doubles_sketch::deserialize(is);
+ REQUIRE(compact_sketch.get_num_retained() == compact_sketch_from_java.get_num_retained());
+ REQUIRE(compact_sketch.get_theta() == Approx(compact_sketch_from_java.get_theta()).margin(1e-10));
+ REQUIRE(compact_sketch.get_estimate() == Approx(compact_sketch_from_java.get_estimate()).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(1) == Approx(compact_sketch_from_java.get_lower_bound(1)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(1) == Approx(compact_sketch_from_java.get_upper_bound(1)).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(2) == Approx(compact_sketch_from_java.get_lower_bound(2)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(2) == Approx(compact_sketch_from_java.get_upper_bound(2)).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(3) == Approx(compact_sketch_from_java.get_lower_bound(3)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(3) == Approx(compact_sketch_from_java.get_upper_bound(3)).margin(1e-10));
+
+ // sketch from Java is not ordered
+ // transform it to ordered so that iteration sequence would match exactly
+ compact_array_of_doubles_sketch ordered_sketch_from_java(compact_sketch_from_java, true);
+ auto it = ordered_sketch_from_java.begin();
+ for (const auto& entry: compact_sketch) {
+ REQUIRE(entry == *it);
+ ++it;
+ }
+}
+
+TEST_CASE("aod sketch: serialization compatibility with java - exact mode with two values", "[tuple_sketch]") {
+ auto update_sketch = update_array_of_doubles_sketch::builder(2).build();
+ std::vector<double> a = {1, 2};
+ for (int i = 0; i < 1000; ++i) update_sketch.update(i, a.data()); // pass vector as pointer
+ auto compact_sketch = update_sketch.compact();
+ REQUIRE_FALSE(compact_sketch.is_estimation_mode());
+
+ // read binary sketch from Java
+ std::ifstream is;
+ is.exceptions(std::ios::failbit | std::ios::badbit);
+ is.open(inputPath + "aod_2_compact_exact_from_java.sk", std::ios::binary);
+ auto compact_sketch_from_java = compact_array_of_doubles_sketch::deserialize(is);
+ REQUIRE(compact_sketch.get_num_retained() == compact_sketch_from_java.get_num_retained());
+ REQUIRE(compact_sketch.get_theta() == Approx(compact_sketch_from_java.get_theta()).margin(1e-10));
+ REQUIRE(compact_sketch.get_estimate() == Approx(compact_sketch_from_java.get_estimate()).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(1) == Approx(compact_sketch_from_java.get_lower_bound(1)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(1) == Approx(compact_sketch_from_java.get_upper_bound(1)).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(2) == Approx(compact_sketch_from_java.get_lower_bound(2)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(2) == Approx(compact_sketch_from_java.get_upper_bound(2)).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(3) == Approx(compact_sketch_from_java.get_lower_bound(3)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(3) == Approx(compact_sketch_from_java.get_upper_bound(3)).margin(1e-10));
+
+ // sketch from Java is not ordered
+ // transform it to ordered so that iteration sequence would match exactly
+ compact_array_of_doubles_sketch ordered_sketch_from_java(compact_sketch_from_java, true);
+ auto it = ordered_sketch_from_java.begin();
+ for (const auto& entry: compact_sketch) {
+ REQUIRE(entry.first == (*it).first);
+ REQUIRE(entry.second.size() == 2);
+ REQUIRE(entry.second[0] == (*it).second[0]);
+ REQUIRE(entry.second[1] == (*it).second[1]);
+ ++it;
+ }
+}
+
+TEST_CASE("aod sketch: stream serialize deserialize - estimation mode", "[tuple_sketch]") {
+ auto update_sketch = update_array_of_doubles_sketch::builder(2).build();
+ std::vector<double> a = {1, 2};
+ for (int i = 0; i < 8192; ++i) update_sketch.update(i, a);
+ auto compact_sketch = update_sketch.compact();
+
+ std::stringstream ss;
+ ss.exceptions(std::ios::failbit | std::ios::badbit);
+ compact_sketch.serialize(ss);
+ auto deserialized_sketch = compact_array_of_doubles_sketch::deserialize(ss);
+ REQUIRE(compact_sketch.get_num_retained() == deserialized_sketch.get_num_retained());
+ REQUIRE(compact_sketch.get_theta() == Approx(deserialized_sketch.get_theta()).margin(1e-10));
+ REQUIRE(compact_sketch.get_estimate() == Approx(deserialized_sketch.get_estimate()).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(1) == Approx(deserialized_sketch.get_lower_bound(1)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(1) == Approx(deserialized_sketch.get_upper_bound(1)).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(2) == Approx(deserialized_sketch.get_lower_bound(2)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(2) == Approx(deserialized_sketch.get_upper_bound(2)).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(3) == Approx(deserialized_sketch.get_lower_bound(3)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(3) == Approx(deserialized_sketch.get_upper_bound(3)).margin(1e-10));
+ // sketches must be ordered and the iteration sequence must match exactly
+ auto it = deserialized_sketch.begin();
+ for (const auto& entry: compact_sketch) {
+ REQUIRE(entry.first == (*it).first);
+ REQUIRE(entry.second.size() == 2);
+ REQUIRE(entry.second[0] == (*it).second[0]);
+ REQUIRE(entry.second[1] == (*it).second[1]);
+ ++it;
+ }
+}
+
+TEST_CASE("aod sketch: bytes to stream serialize deserialize - estimation mode", "[tuple_sketch]") {
+ auto update_sketch = update_array_of_doubles_sketch::builder(2).build();
+ std::vector<double> a = {1, 2};
+ for (int i = 0; i < 8192; ++i) update_sketch.update(i, a);
+ auto compact_sketch = update_sketch.compact();
+
+ auto bytes = compact_sketch.serialize();
+ std::stringstream ss;
+ ss.exceptions(std::ios::failbit | std::ios::badbit);
+ ss.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+ auto deserialized_sketch = compact_array_of_doubles_sketch::deserialize(ss);
+ REQUIRE(compact_sketch.get_num_retained() == deserialized_sketch.get_num_retained());
+ REQUIRE(compact_sketch.get_theta() == Approx(deserialized_sketch.get_theta()).margin(1e-10));
+ REQUIRE(compact_sketch.get_estimate() == Approx(deserialized_sketch.get_estimate()).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(1) == Approx(deserialized_sketch.get_lower_bound(1)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(1) == Approx(deserialized_sketch.get_upper_bound(1)).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(2) == Approx(deserialized_sketch.get_lower_bound(2)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(2) == Approx(deserialized_sketch.get_upper_bound(2)).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(3) == Approx(deserialized_sketch.get_lower_bound(3)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(3) == Approx(deserialized_sketch.get_upper_bound(3)).margin(1e-10));
+ // sketches must be ordered and the iteration sequence must match exactly
+ auto it = deserialized_sketch.begin();
+ for (const auto& entry: compact_sketch) {
+ REQUIRE(entry.first == (*it).first);
+ REQUIRE(entry.second.size() == 2);
+ REQUIRE(entry.second[0] == (*it).second[0]);
+ REQUIRE(entry.second[1] == (*it).second[1]);
+ ++it;
+ }
+}
+
+TEST_CASE("aod sketch: bytes serialize deserialize - estimation mode", "[tuple_sketch]") {
+ auto update_sketch = update_array_of_doubles_sketch::builder(2).build();
+ std::vector<double> a = {1, 2};
+ for (int i = 0; i < 8192; ++i) update_sketch.update(i, a);
+ auto compact_sketch = update_sketch.compact();
+
+ auto bytes = compact_sketch.serialize();
+ auto deserialized_sketch = compact_array_of_doubles_sketch::deserialize(bytes.data(), bytes.size());
+ REQUIRE(compact_sketch.get_num_retained() == deserialized_sketch.get_num_retained());
+ REQUIRE(compact_sketch.get_theta() == Approx(deserialized_sketch.get_theta()).margin(1e-10));
+ REQUIRE(compact_sketch.get_estimate() == Approx(deserialized_sketch.get_estimate()).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(1) == Approx(deserialized_sketch.get_lower_bound(1)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(1) == Approx(deserialized_sketch.get_upper_bound(1)).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(2) == Approx(deserialized_sketch.get_lower_bound(2)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(2) == Approx(deserialized_sketch.get_upper_bound(2)).margin(1e-10));
+ REQUIRE(compact_sketch.get_lower_bound(3) == Approx(deserialized_sketch.get_lower_bound(3)).margin(1e-10));
+ REQUIRE(compact_sketch.get_upper_bound(3) == Approx(deserialized_sketch.get_upper_bound(3)).margin(1e-10));
+ // sketches must be ordered and the iteration sequence must match exactly
+ auto it = deserialized_sketch.begin();
+ for (const auto& entry: compact_sketch) {
+ REQUIRE(entry.first == (*it).first);
+ REQUIRE(entry.second.size() == 2);
+ REQUIRE(entry.second[0] == (*it).second[0]);
+ REQUIRE(entry.second[1] == (*it).second[1]);
+ ++it;
+ }
+}
+
+TEST_CASE("aod union: half overlap", "[tuple_sketch]") {
+ std::vector<double> a = {1};
+
+ auto update_sketch1 = update_array_of_doubles_sketch::builder().build();
+ for (int i = 0; i < 1000; ++i) update_sketch1.update(i, a);
+
+ auto update_sketch2 = update_array_of_doubles_sketch::builder().build();
+ for (int i = 500; i < 1500; ++i) update_sketch2.update(i, a);
+
+ auto u = array_of_doubles_union::builder().build();
+ u.update(update_sketch1);
+ u.update(update_sketch2);
+ auto result = u.get_result();
+ REQUIRE(result.get_estimate() == Approx(1500).margin(0.01));
+}
+
+TEST_CASE("aod intersection: half overlap", "[tuple_sketch]") {
+ std::vector<double> a = {1};
+
+ auto update_sketch1 = update_array_of_doubles_sketch::builder().build();
+ for (int i = 0; i < 1000; ++i) update_sketch1.update(i, a);
+
+ auto update_sketch2 = update_array_of_doubles_sketch::builder().build();
+ for (int i = 500; i < 1500; ++i) update_sketch2.update(i, a);
+
+ array_of_doubles_intersection<array_of_doubles_union_policy> intersection;
+ intersection.update(update_sketch1);
+ intersection.update(update_sketch2);
+ auto result = intersection.get_result();
+ REQUIRE(result.get_estimate() == Approx(500).margin(0.01));
+}
+
+} /* namespace datasketches */
diff --git a/tuple/test/theta_a_not_b_experimental_test.cpp b/tuple/test/theta_a_not_b_experimental_test.cpp
new file mode 100644
index 0000000..6b44f8b
--- /dev/null
+++ b/tuple/test/theta_a_not_b_experimental_test.cpp
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch.hpp>
+
+#include <theta_a_not_b_experimental.hpp>
+
+namespace datasketches {
+
+// These tests have been copied from the existing theta sketch implementation.
+
+using update_theta_sketch = update_theta_sketch_experimental<>;
+using compact_theta_sketch = compact_theta_sketch_experimental<>;
+using theta_a_not_b = theta_a_not_b_experimental<>;
+
+TEST_CASE("theta a-not-b: empty", "[theta_a_not_b]") {
+ theta_a_not_b a_not_b;
+ auto a = update_theta_sketch::builder().build();
+ auto b = update_theta_sketch::builder().build();
+ compact_theta_sketch result = a_not_b.compute(a, b);
+ REQUIRE(result.get_num_retained() == 0);
+ REQUIRE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("theta a-not-b: non empty no retained keys", "[theta_a_not_b]") {
+ update_theta_sketch a = update_theta_sketch::builder().build();
+ a.update(1);
+ update_theta_sketch b = update_theta_sketch::builder().set_p(0.001).build();
+ theta_a_not_b a_not_b;
+
+ // B is still empty
+ compact_theta_sketch result = a_not_b.compute(a, b);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_num_retained() == 1);
+ REQUIRE(result.get_theta() == Approx(1).margin(1e-10));
+ REQUIRE(result.get_estimate() == 1.0);
+
+ // B is not empty in estimation mode and no entries
+ b.update(1);
+ REQUIRE(b.get_num_retained() == 0U);
+
+ result = a_not_b.compute(a, b);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_num_retained() == 0);
+ REQUIRE(result.get_theta() == Approx(0.001).margin(1e-10));
+ REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("theta a-not-b: exact mode half overlap", "[theta_a_not_b]") {
+ update_theta_sketch a = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 1000; i++) a.update(value++);
+
+ update_theta_sketch b = update_theta_sketch::builder().build();
+ value = 500;
+ for (int i = 0; i < 1000; i++) b.update(value++);
+
+ theta_a_not_b a_not_b;
+
+ // unordered inputs, ordered result
+ compact_theta_sketch result = a_not_b.compute(a, b);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.is_ordered());
+ REQUIRE(result.get_estimate() == 500.0);
+
+ // unordered inputs, unordered result
+ result = a_not_b.compute(a, b, false);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE_FALSE(result.is_ordered());
+ REQUIRE(result.get_estimate() == 500.0);
+
+ // ordered inputs
+ result = a_not_b.compute(a.compact(), b.compact());
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.is_ordered());
+ REQUIRE(result.get_estimate() == 500.0);
+
+ // A is ordered, so the result is ordered regardless
+ result = a_not_b.compute(a.compact(), b, false);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.is_ordered());
+ REQUIRE(result.get_estimate() == 500.0);
+}
+
+TEST_CASE("theta a-not-b: exact mode disjoint", "[theta_a_not_b]") {
+ update_theta_sketch a = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 1000; i++) a.update(value++);
+
+ update_theta_sketch b = update_theta_sketch::builder().build();
+ for (int i = 0; i < 1000; i++) b.update(value++);
+
+ theta_a_not_b a_not_b;
+
+ // unordered inputs
+ compact_theta_sketch result = a_not_b.compute(a, b);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 1000.0);
+
+ // ordered inputs
+ result = a_not_b.compute(a.compact(), b.compact());
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 1000.0);
+}
+
+TEST_CASE("theta a-not-b: exact mode full overlap", "[theta_a_not_b]") {
+ update_theta_sketch sketch = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 1000; i++) sketch.update(value++);
+
+ theta_a_not_b a_not_b;
+
+ // unordered inputs
+ compact_theta_sketch result = a_not_b.compute(sketch, sketch);
+ REQUIRE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+
+ // ordered inputs
+ result = a_not_b.compute(sketch.compact(), sketch.compact());
+ REQUIRE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("theta a-not-b: estimation mode half overlap", "[theta_a_not_b]") {
+ update_theta_sketch a = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; i++) a.update(value++);
+
+ update_theta_sketch b = update_theta_sketch::builder().build();
+ value = 5000;
+ for (int i = 0; i < 10000; i++) b.update(value++);
+
+ theta_a_not_b a_not_b;
+
+ // unordered inputs
+ compact_theta_sketch result = a_not_b.compute(a, b);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.02));
+
+ // ordered inputs
+ result = a_not_b.compute(a.compact(), b.compact());
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.02));
+}
+
+TEST_CASE("theta a-not-b: estimation mode disjoint", "[theta_a_not_b]") {
+ update_theta_sketch a = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; i++) a.update(value++);
+
+ update_theta_sketch b = update_theta_sketch::builder().build();
+ for (int i = 0; i < 10000; i++) b.update(value++);
+
+ theta_a_not_b a_not_b;
+
+ // unordered inputs
+ compact_theta_sketch result = a_not_b.compute(a, b);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(10000).margin(10000 * 0.02));
+
+ // ordered inputs
+ result = a_not_b.compute(a.compact(), b.compact());
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(10000).margin(10000 * 0.02));
+}
+
+TEST_CASE("theta a-not-b: estimation mode full overlap", "[theta_a_not_b]") {
+ update_theta_sketch sketch = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; i++) sketch.update(value++);
+
+ theta_a_not_b a_not_b;
+
+ // unordered inputs
+ compact_theta_sketch result = a_not_b.compute(sketch, sketch);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+
+ // ordered inputs
+ result = a_not_b.compute(sketch.compact(), sketch.compact());
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("theta a-not-b: seed mismatch", "[theta_a_not_b]") {
+ update_theta_sketch sketch = update_theta_sketch::builder().build();
+ sketch.update(1); // non-empty should not be ignored
+ theta_a_not_b a_not_b(123);
+ REQUIRE_THROWS_AS(a_not_b.compute(sketch, sketch), std::invalid_argument);
+}
+
+TEST_CASE("theta a-not-b: issue #152", "[theta_a_not_b]") {
+ update_theta_sketch a = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; i++) a.update(value++);
+
+ update_theta_sketch b = update_theta_sketch::builder().build();
+ value = 5000;
+ for (int i = 0; i < 25000; i++) b.update(value++);
+
+ theta_a_not_b a_not_b;
+
+ // unordered inputs
+ compact_theta_sketch result = a_not_b.compute(a, b);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.03));
+
+ // ordered inputs
+ result = a_not_b.compute(a.compact(), b.compact());
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.03));
+}
+
+} /* namespace datasketches */
diff --git a/tuple/test/theta_compact_empty_from_java.sk b/tuple/test/theta_compact_empty_from_java.sk
new file mode 100644
index 0000000..f6c647f
--- /dev/null
+++ b/tuple/test/theta_compact_empty_from_java.sk
Binary files differ
diff --git a/tuple/test/theta_compact_estimation_from_java.sk b/tuple/test/theta_compact_estimation_from_java.sk
new file mode 100644
index 0000000..7c6babf
--- /dev/null
+++ b/tuple/test/theta_compact_estimation_from_java.sk
Binary files differ
diff --git a/tuple/test/theta_compact_single_item_from_java.sk b/tuple/test/theta_compact_single_item_from_java.sk
new file mode 100644
index 0000000..be5ee68
--- /dev/null
+++ b/tuple/test/theta_compact_single_item_from_java.sk
Binary files differ
diff --git a/tuple/test/theta_intersection_experimental_test.cpp b/tuple/test/theta_intersection_experimental_test.cpp
new file mode 100644
index 0000000..3337636
--- /dev/null
+++ b/tuple/test/theta_intersection_experimental_test.cpp
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <catch.hpp>
+
+#include <theta_intersection_experimental.hpp>
+
+namespace datasketches {
+
+// These tests have been copied from the existing theta sketch implementation.
+
+using update_theta_sketch = update_theta_sketch_experimental<>;
+using compact_theta_sketch = compact_theta_sketch_experimental<>;
+using theta_intersection = theta_intersection_experimental<>;
+
+TEST_CASE("theta intersection: invalid", "[theta_intersection]") {
+ theta_intersection intersection;
+ REQUIRE_FALSE(intersection.has_result());
+ REQUIRE_THROWS_AS(intersection.get_result(), std::invalid_argument);
+}
+
+TEST_CASE("theta intersection: empty", "[theta_intersection]") {
+ theta_intersection intersection;
+ update_theta_sketch sketch = update_theta_sketch::builder().build();
+ intersection.update(sketch);
+ compact_theta_sketch result = intersection.get_result();
+ REQUIRE(result.get_num_retained() == 0);
+ REQUIRE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+
+ intersection.update(sketch);
+ result = intersection.get_result();
+ REQUIRE(result.get_num_retained() == 0);
+ REQUIRE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("theta intersection: non empty no retained keys", "[theta_intersection]") {
+ update_theta_sketch sketch = update_theta_sketch::builder().set_p(0.001).build();
+ sketch.update(1);
+ theta_intersection intersection;
+ intersection.update(sketch);
+ compact_theta_sketch result = intersection.get_result();
+ REQUIRE(result.get_num_retained() == 0);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_theta() == Approx(0.001).margin(1e-10));
+ REQUIRE(result.get_estimate() == 0.0);
+
+ intersection.update(sketch);
+ result = intersection.get_result();
+ REQUIRE(result.get_num_retained() == 0);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_theta() == Approx(0.001).margin(1e-10));
+ REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("theta intersection: exact mode half overlap unordered", "[theta_intersection]") {
+ update_theta_sketch sketch1 = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 1000; i++) sketch1.update(value++);
+
+ update_theta_sketch sketch2 = update_theta_sketch::builder().build();
+ value = 500;
+ for (int i = 0; i < 1000; i++) sketch2.update(value++);
+
+ theta_intersection intersection;
+ intersection.update(sketch1);
+ intersection.update(sketch2);
+ compact_theta_sketch result = intersection.get_result();
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 500.0);
+}
+
+TEST_CASE("theta intersection: exact mode half overlap ordered", "[theta_intersection]") {
+ update_theta_sketch sketch1 = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 1000; i++) sketch1.update(value++);
+
+ update_theta_sketch sketch2 = update_theta_sketch::builder().build();
+ value = 500;
+ for (int i = 0; i < 1000; i++) sketch2.update(value++);
+
+ theta_intersection intersection;
+ intersection.update(sketch1.compact());
+ intersection.update(sketch2.compact());
+ compact_theta_sketch result = intersection.get_result();
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 500.0);
+}
+
+TEST_CASE("theta intersection: exact mode disjoint unordered", "[theta_intersection]") {
+ update_theta_sketch sketch1 = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 1000; i++) sketch1.update(value++);
+
+ update_theta_sketch sketch2 = update_theta_sketch::builder().build();
+ for (int i = 0; i < 1000; i++) sketch2.update(value++);
+
+ theta_intersection intersection;
+ intersection.update(sketch1);
+ intersection.update(sketch2);
+ compact_theta_sketch result = intersection.get_result();
+ REQUIRE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("theta intersection: exact mode disjoint ordered", "[theta_intersection]") {
+ update_theta_sketch sketch1 = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 1000; i++) sketch1.update(value++);
+
+ update_theta_sketch sketch2 = update_theta_sketch::builder().build();
+ for (int i = 0; i < 1000; i++) sketch2.update(value++);
+
+ theta_intersection intersection;
+ intersection.update(sketch1.compact());
+ intersection.update(sketch2.compact());
+ compact_theta_sketch result = intersection.get_result();
+ REQUIRE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("theta intersection: estimation mode half overlap unordered", "[theta_intersection]") {
+ update_theta_sketch sketch1 = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; i++) sketch1.update(value++);
+
+ update_theta_sketch sketch2 = update_theta_sketch::builder().build();
+ value = 5000;
+ for (int i = 0; i < 10000; i++) sketch2.update(value++);
+
+ theta_intersection intersection;
+ intersection.update(sketch1);
+ intersection.update(sketch2);
+ compact_theta_sketch result = intersection.get_result();
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.02));
+}
+
+TEST_CASE("theta intersection: estimation mode half overlap ordered", "[theta_intersection]") {
+ update_theta_sketch sketch1 = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; i++) sketch1.update(value++);
+
+ update_theta_sketch sketch2 = update_theta_sketch::builder().build();
+ value = 5000;
+ for (int i = 0; i < 10000; i++) sketch2.update(value++);
+
+ theta_intersection intersection;
+ intersection.update(sketch1.compact());
+ intersection.update(sketch2.compact());
+ compact_theta_sketch result = intersection.get_result();
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.02));
+}
+
+TEST_CASE("theta intersection: estimation mode disjoint unordered", "[theta_intersection]") {
+ update_theta_sketch sketch1 = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; i++) sketch1.update(value++);
+
+ update_theta_sketch sketch2 = update_theta_sketch::builder().build();
+ for (int i = 0; i < 10000; i++) sketch2.update(value++);
+
+ theta_intersection intersection;
+ intersection.update(sketch1);
+ intersection.update(sketch2);
+ compact_theta_sketch result = intersection.get_result();
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("theta intersection: estimation mode disjoint ordered", "[theta_intersection]") {
+ update_theta_sketch sketch1 = update_theta_sketch::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; i++) sketch1.update(value++);
+
+ update_theta_sketch sketch2 = update_theta_sketch::builder().build();
+ for (int i = 0; i < 10000; i++) sketch2.update(value++);
+
+ theta_intersection intersection;
+ intersection.update(sketch1.compact());
+ intersection.update(sketch2.compact());
+ compact_theta_sketch result = intersection.get_result();
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("theta intersection: seed mismatch", "[theta_intersection]") {
+ update_theta_sketch sketch = update_theta_sketch::builder().build();
+ sketch.update(1); // non-empty should not be ignored
+ theta_intersection intersection(123);
+ REQUIRE_THROWS_AS(intersection.update(sketch), std::invalid_argument);
+}
+
+} /* namespace datasketches */
diff --git a/tuple/test/theta_jaccard_similarity_test.cpp b/tuple/test/theta_jaccard_similarity_test.cpp
new file mode 100644
index 0000000..fda1a6d
--- /dev/null
+++ b/tuple/test/theta_jaccard_similarity_test.cpp
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <iostream>
+
+#include <catch.hpp>
+#include <jaccard_similarity.hpp>
+
+namespace datasketches {
+
+using update_theta_sketch = update_theta_sketch_experimental<>;
+
+TEST_CASE("theta jaccard: empty", "[theta_sketch]") {
+ auto sk_a = update_theta_sketch::builder().build();
+ auto sk_b = update_theta_sketch::builder().build();
+ auto jc = theta_jaccard_similarity::jaccard(sk_a, sk_b);
+ REQUIRE(jc == std::array<double, 3>{1, 1, 1});
+}
+
+TEST_CASE("theta jaccard: same sketch exact mode", "[theta_sketch]") {
+ auto sk = update_theta_sketch::builder().build();
+ for (int i = 0; i < 1000; ++i) sk.update(i);
+
+ // update sketch
+ auto jc = theta_jaccard_similarity::jaccard(sk, sk);
+ REQUIRE(jc == std::array<double, 3>{1, 1, 1});
+
+ // compact sketch
+ jc = theta_jaccard_similarity::jaccard(sk.compact(), sk.compact());
+ REQUIRE(jc == std::array<double, 3>{1, 1, 1});
+}
+
+TEST_CASE("theta jaccard: full overlap exact mode", "[theta_sketch]") {
+ auto sk_a = update_theta_sketch::builder().build();
+ auto sk_b = update_theta_sketch::builder().build();
+ for (int i = 0; i < 1000; ++i) {
+ sk_a.update(i);
+ sk_b.update(i);
+ }
+
+ // update sketches
+ auto jc = theta_jaccard_similarity::jaccard(sk_a, sk_b);
+ REQUIRE(jc == std::array<double, 3>{1, 1, 1});
+
+ // compact sketches
+ jc = theta_jaccard_similarity::jaccard(sk_a.compact(), sk_b.compact());
+ REQUIRE(jc == std::array<double, 3>{1, 1, 1});
+}
+
+TEST_CASE("theta jaccard: disjoint exact mode", "[theta_sketch]") {
+ auto sk_a = update_theta_sketch::builder().build();
+ auto sk_b = update_theta_sketch::builder().build();
+ for (int i = 0; i < 1000; ++i) {
+ sk_a.update(i);
+ sk_b.update(i + 1000);
+ }
+
+ // update sketches
+ auto jc = theta_jaccard_similarity::jaccard(sk_a, sk_b);
+ REQUIRE(jc == std::array<double, 3>{0, 0, 0});
+
+ // compact sketches
+ jc = theta_jaccard_similarity::jaccard(sk_a.compact(), sk_b.compact());
+ REQUIRE(jc == std::array<double, 3>{0, 0, 0});
+}
+
+TEST_CASE("theta jaccard: half overlap estimation mode", "[theta_sketch]") {
+ auto sk_a = update_theta_sketch::builder().build();
+ auto sk_b = update_theta_sketch::builder().build();
+ for (int i = 0; i < 10000; ++i) {
+ sk_a.update(i);
+ sk_b.update(i + 5000);
+ }
+
+ // update sketches
+ auto jc = theta_jaccard_similarity::jaccard(sk_a, sk_b);
+ REQUIRE(jc[0] == Approx(0.33).margin(0.01));
+ REQUIRE(jc[1] == Approx(0.33).margin(0.01));
+ REQUIRE(jc[2] == Approx(0.33).margin(0.01));
+
+ // compact sketches
+ jc = theta_jaccard_similarity::jaccard(sk_a.compact(), sk_b.compact());
+ REQUIRE(jc[0] == Approx(0.33).margin(0.01));
+ REQUIRE(jc[1] == Approx(0.33).margin(0.01));
+ REQUIRE(jc[2] == Approx(0.33).margin(0.01));
+}
+
+/**
+ * The distribution is quite tight, about +/- 0.7%, which is pretty good since the accuracy of the
+ * underlying sketch is about +/- 1.56%.
+ */
+TEST_CASE("theta jaccard: similarity test", "[theta_sketch]") {
+ const int8_t min_lg_k = 12;
+ const int u1 = 1 << 20;
+ const int u2 = u1 * 0.95;
+ const double threshold = 0.943;
+
+ auto expected = update_theta_sketch::builder().set_lg_k(min_lg_k).build();
+ for (int i = 0; i < u1; ++i) expected.update(i);
+
+ auto actual = update_theta_sketch::builder().set_lg_k(min_lg_k).build();
+ for (int i = 0; i < u2; ++i) actual.update(i);
+
+ REQUIRE(theta_jaccard_similarity::similarity_test(actual, expected, threshold));
+ REQUIRE(theta_jaccard_similarity::similarity_test(actual, actual, threshold));
+}
+
+/**
+ * The distribution is much looser here, about +/- 14%. This is due to the fact that intersections loose accuracy
+ * as the ratio of intersection to the union becomes a small number.
+ */
+TEST_CASE("theta jaccard: dissimilarity test", "[theta_sketch]") {
+ const int8_t min_lg_k = 12;
+ const int u1 = 1 << 20;
+ const int u2 = u1 * 0.05;
+ const double threshold = 0.061;
+
+ auto expected = update_theta_sketch::builder().set_lg_k(min_lg_k).build();
+ for (int i = 0; i < u1; ++i) expected.update(i);
+
+ auto actual = update_theta_sketch::builder().set_lg_k(min_lg_k).build();
+ for (int i = 0; i < u2; ++i) actual.update(i);
+
+ REQUIRE(theta_jaccard_similarity::dissimilarity_test(actual, expected, threshold));
+ REQUIRE_FALSE(theta_jaccard_similarity::dissimilarity_test(actual, actual, threshold));
+}
+
+} /* namespace datasketches */
diff --git a/tuple/test/theta_sketch_experimental_test.cpp b/tuple/test/theta_sketch_experimental_test.cpp
new file mode 100644
index 0000000..61435a4
--- /dev/null
+++ b/tuple/test/theta_sketch_experimental_test.cpp
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <fstream>
+#include <sstream>
+
+#include <catch.hpp>
+#include <theta_sketch_experimental.hpp>
+
+namespace datasketches {
+
+#ifdef TEST_BINARY_INPUT_PATH
+const std::string inputPath = TEST_BINARY_INPUT_PATH;
+#else
+const std::string inputPath = "test/";
+#endif
+
+// These tests have been copied from the existing theta sketch implementation.
+// Serialization as base class and serialization of update sketch have been removed.
+
+using update_theta_sketch = update_theta_sketch_experimental<>;
+using compact_theta_sketch = compact_theta_sketch_experimental<>;
+
+TEST_CASE("theta sketch: empty", "[theta_sketch]") {
+ update_theta_sketch update_sketch = update_theta_sketch::builder().build();
+ REQUIRE(update_sketch.is_empty());
+ REQUIRE_FALSE(update_sketch.is_estimation_mode());
+ REQUIRE(update_sketch.get_theta() == 1.0);
+ REQUIRE(update_sketch.get_estimate() == 0.0);
+ REQUIRE(update_sketch.get_lower_bound(1) == 0.0);
+ REQUIRE(update_sketch.get_upper_bound(1) == 0.0);
+
+ compact_theta_sketch compact_sketch = update_sketch.compact();
+ REQUIRE(compact_sketch.is_empty());
+ REQUIRE_FALSE(compact_sketch.is_estimation_mode());
+ REQUIRE(compact_sketch.get_theta() == 1.0);
+ REQUIRE(compact_sketch.get_estimate() == 0.0);
+ REQUIRE(compact_sketch.get_lower_bound(1) == 0.0);
+ REQUIRE(compact_sketch.get_upper_bound(1) == 0.0);
+}
+
+TEST_CASE("theta sketch: non empty no retained keys", "[theta_sketch]") {
+ update_theta_sketch update_sketch = update_theta_sketch::builder().set_p(0.001).build();
+ update_sketch.update(1);
+ //std::cerr << update_sketch.to_string();
+ REQUIRE(update_sketch.get_num_retained() == 0);
+ REQUIRE_FALSE(update_sketch.is_empty());
+ REQUIRE(update_sketch.is_estimation_mode());
+ REQUIRE(update_sketch.get_estimate() == 0.0);
+ REQUIRE(update_sketch.get_lower_bound(1) == 0.0);
+ REQUIRE(update_sketch.get_upper_bound(1) > 0);
+
+ compact_theta_sketch compact_sketch = update_sketch.compact();
+ REQUIRE(compact_sketch.get_num_retained() == 0);
+ REQUIRE_FALSE(compact_sketch.is_empty());
+ REQUIRE(compact_sketch.is_estimation_mode());
+ REQUIRE(compact_sketch.get_estimate() == 0.0);
+ REQUIRE(compact_sketch.get_lower_bound(1) == 0.0);
+ REQUIRE(compact_sketch.get_upper_bound(1) > 0);
+}
+
+TEST_CASE("theta sketch: single item", "[theta_sketch]") {
+ update_theta_sketch update_sketch = update_theta_sketch::builder().build();
+ update_sketch.update(1);
+ REQUIRE_FALSE(update_sketch.is_empty());
+ REQUIRE_FALSE(update_sketch.is_estimation_mode());
+ REQUIRE(update_sketch.get_theta() == 1.0);
+ REQUIRE(update_sketch.get_estimate() == 1.0);
+ REQUIRE(update_sketch.get_lower_bound(1) == 1.0);
+ REQUIRE(update_sketch.get_upper_bound(1) == 1.0);
+
+ compact_theta_sketch compact_sketch = update_sketch.compact();
+ REQUIRE_FALSE(compact_sketch.is_empty());
+ REQUIRE_FALSE(compact_sketch.is_estimation_mode());
+ REQUIRE(compact_sketch.get_theta() == 1.0);
+ REQUIRE(compact_sketch.get_estimate() == 1.0);
+ REQUIRE(compact_sketch.get_lower_bound(1) == 1.0);
+ REQUIRE(compact_sketch.get_upper_bound(1) == 1.0);
+}
+
+TEST_CASE("theta sketch: resize exact", "[theta_sketch]") {
+ update_theta_sketch update_sketch = update_theta_sketch::builder().build();
+ for (int i = 0; i < 2000; i++) update_sketch.update(i);
+ REQUIRE_FALSE(update_sketch.is_empty());
+ REQUIRE_FALSE(update_sketch.is_estimation_mode());
+ REQUIRE(update_sketch.get_theta() == 1.0);
+ REQUIRE(update_sketch.get_estimate() == 2000.0);
+ REQUIRE(update_sketch.get_lower_bound(1) == 2000.0);
+ REQUIRE(update_sketch.get_upper_bound(1) == 2000.0);
+
+ compact_theta_sketch compact_sketch = update_sketch.compact();
+ REQUIRE_FALSE(compact_sketch.is_empty());
+ REQUIRE_FALSE(compact_sketch.is_estimation_mode());
+ REQUIRE(compact_sketch.get_theta() == 1.0);
+ REQUIRE(compact_sketch.get_estimate() == 2000.0);
+ REQUIRE(compact_sketch.get_lower_bound(1) == 2000.0);
+ REQUIRE(compact_sketch.get_upper_bound(1) == 2000.0);
+}
+
+TEST_CASE("theta sketch: estimation", "[theta_sketch]") {
+ update_theta_sketch update_sketch = update_theta_sketch::builder().set_resize_factor(update_theta_sketch::resize_factor::X1).build();
+ const int n = 8000;
+ for (int i = 0; i < n; i++) update_sketch.update(i);
+ //std::cerr << update_sketch.to_string();
+ REQUIRE_FALSE(update_sketch.is_empty());
+ REQUIRE(update_sketch.is_estimation_mode());
+ REQUIRE(update_sketch.get_theta() < 1.0);
+ REQUIRE(update_sketch.get_estimate() == Approx((double) n).margin(n * 0.01));
+ REQUIRE(update_sketch.get_lower_bound(1) < n);
+ REQUIRE(update_sketch.get_upper_bound(1) > n);
+
+ const uint32_t k = 1 << update_theta_sketch::builder::DEFAULT_LG_K;
+ REQUIRE(update_sketch.get_num_retained() >= k);
+ update_sketch.trim();
+ REQUIRE(update_sketch.get_num_retained() == k);
+
+ compact_theta_sketch compact_sketch = update_sketch.compact();
+ REQUIRE_FALSE(compact_sketch.is_empty());
+ REQUIRE(compact_sketch.is_ordered());
+ REQUIRE(compact_sketch.is_estimation_mode());
+ REQUIRE(compact_sketch.get_theta() < 1.0);
+ REQUIRE(compact_sketch.get_estimate() == Approx((double) n).margin(n * 0.01));
+ REQUIRE(compact_sketch.get_lower_bound(1) < n);
+ REQUIRE(compact_sketch.get_upper_bound(1) > n);
+}
+
+TEST_CASE("theta sketch: deserialize compact empty from java", "[theta_sketch]") {
+ std::ifstream is;
+ is.exceptions(std::ios::failbit | std::ios::badbit);
+ is.open(inputPath + "theta_compact_empty_from_java.sk", std::ios::binary);
+ auto sketch = compact_theta_sketch::deserialize(is);
+ REQUIRE(sketch.is_empty());
+ REQUIRE_FALSE(sketch.is_estimation_mode());
+ REQUIRE(sketch.get_num_retained() == 0);
+ REQUIRE(sketch.get_theta() == 1.0);
+ REQUIRE(sketch.get_estimate() == 0.0);
+ REQUIRE(sketch.get_lower_bound(1) == 0.0);
+ REQUIRE(sketch.get_upper_bound(1) == 0.0);
+}
+
+TEST_CASE("theta sketch: deserialize single item from java", "[theta_sketch]") {
+ std::ifstream is;
+ is.exceptions(std::ios::failbit | std::ios::badbit);
+ is.open(inputPath + "theta_compact_single_item_from_java.sk", std::ios::binary);
+ auto sketch = compact_theta_sketch::deserialize(is);
+ REQUIRE_FALSE(sketch.is_empty());
+ REQUIRE_FALSE(sketch.is_estimation_mode());
+ REQUIRE(sketch.get_num_retained() == 1);
+ REQUIRE(sketch.get_theta() == 1.0);
+ REQUIRE(sketch.get_estimate() == 1.0);
+ REQUIRE(sketch.get_lower_bound(1) == 1.0);
+ REQUIRE(sketch.get_upper_bound(1) == 1.0);
+}
+
+TEST_CASE("theta sketch: deserialize compact estimation from java", "[theta_sketch]") {
+ std::ifstream is;
+ is.exceptions(std::ios::failbit | std::ios::badbit);
+ is.open(inputPath + "theta_compact_estimation_from_java.sk", std::ios::binary);
+ auto sketch = compact_theta_sketch::deserialize(is);
+ REQUIRE_FALSE(sketch.is_empty());
+ REQUIRE(sketch.is_estimation_mode());
+ REQUIRE(sketch.is_ordered());
+ REQUIRE(sketch.get_num_retained() == 4342);
+ REQUIRE(sketch.get_theta() == Approx(0.531700444213199).margin(1e-10));
+ REQUIRE(sketch.get_estimate() == Approx(8166.25234614053).margin(1e-10));
+ REQUIRE(sketch.get_lower_bound(2) == Approx(7996.956955317471).margin(1e-10));
+ REQUIRE(sketch.get_upper_bound(2) == Approx(8339.090301078124).margin(1e-10));
+
+ // the same construction process in Java must have produced exactly the same sketch
+ update_theta_sketch update_sketch = update_theta_sketch::builder().build();
+ const int n = 8192;
+ for (int i = 0; i < n; i++) update_sketch.update(i);
+ REQUIRE(sketch.get_num_retained() == update_sketch.get_num_retained());
+ REQUIRE(sketch.get_theta() == Approx(update_sketch.get_theta()).margin(1e-10));
+ REQUIRE(sketch.get_estimate() == Approx(update_sketch.get_estimate()).margin(1e-10));
+ REQUIRE(sketch.get_lower_bound(1) == Approx(update_sketch.get_lower_bound(1)).margin(1e-10));
+ REQUIRE(sketch.get_upper_bound(1) == Approx(update_sketch.get_upper_bound(1)).margin(1e-10));
+ REQUIRE(sketch.get_lower_bound(2) == Approx(update_sketch.get_lower_bound(2)).margin(1e-10));
+ REQUIRE(sketch.get_upper_bound(2) == Approx(update_sketch.get_upper_bound(2)).margin(1e-10));
+ REQUIRE(sketch.get_lower_bound(3) == Approx(update_sketch.get_lower_bound(3)).margin(1e-10));
+ REQUIRE(sketch.get_upper_bound(3) == Approx(update_sketch.get_upper_bound(3)).margin(1e-10));
+ compact_theta_sketch compact_sketch = update_sketch.compact();
+ // the sketches are ordered, so the iteration sequence must match exactly
+ auto iter = sketch.begin();
+ for (const auto& key: compact_sketch) {
+ REQUIRE(*iter == key);
+ ++iter;
+ }
+}
+
+TEST_CASE("theta sketch: serialize deserialize stream and bytes equivalence", "[theta_sketch]") {
+ update_theta_sketch update_sketch = update_theta_sketch::builder().build();
+ const int n = 8192;
+ for (int i = 0; i < n; i++) update_sketch.update(i);
+
+ std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+ update_sketch.compact().serialize(s);
+ auto bytes = update_sketch.compact().serialize();
+ REQUIRE(bytes.size() == static_cast<size_t>(s.tellp()));
+ for (size_t i = 0; i < bytes.size(); ++i) {
+ REQUIRE(((char*)bytes.data())[i] == (char)s.get());
+ }
+
+ s.seekg(0); // rewind
+ compact_theta_sketch deserialized_sketch1 = compact_theta_sketch::deserialize(s);
+ compact_theta_sketch deserialized_sketch2 = compact_theta_sketch::deserialize(bytes.data(), bytes.size());
+ REQUIRE(bytes.size() == static_cast<size_t>(s.tellg()));
+ REQUIRE(deserialized_sketch2.is_empty() == deserialized_sketch1.is_empty());
+ REQUIRE(deserialized_sketch2.is_ordered() == deserialized_sketch1.is_ordered());
+ REQUIRE(deserialized_sketch2.get_num_retained() == deserialized_sketch1.get_num_retained());
+ REQUIRE(deserialized_sketch2.get_theta() == deserialized_sketch1.get_theta());
+ REQUIRE(deserialized_sketch2.get_estimate() == deserialized_sketch1.get_estimate());
+ REQUIRE(deserialized_sketch2.get_lower_bound(1) == deserialized_sketch1.get_lower_bound(1));
+ REQUIRE(deserialized_sketch2.get_upper_bound(1) == deserialized_sketch1.get_upper_bound(1));
+ // the sketches are ordered, so the iteration sequence must match exactly
+ auto iter = deserialized_sketch1.begin();
+ for (auto key: deserialized_sketch2) {
+ REQUIRE(*iter == key);
+ ++iter;
+ }
+}
+
+TEST_CASE("theta sketch: deserialize compact single item buffer overrun", "[theta_sketch]") {
+ update_theta_sketch update_sketch = update_theta_sketch::builder().build();
+ update_sketch.update(1);
+ auto bytes = update_sketch.compact().serialize();
+ REQUIRE_THROWS_AS(compact_theta_sketch::deserialize(bytes.data(), 7), std::out_of_range);
+ REQUIRE_THROWS_AS(compact_theta_sketch::deserialize(bytes.data(), bytes.size() - 1), std::out_of_range);
+}
+
+} /* namespace datasketches */
diff --git a/tuple/test/theta_union_experimental_test.cpp b/tuple/test/theta_union_experimental_test.cpp
new file mode 100644
index 0000000..c270a11
--- /dev/null
+++ b/tuple/test/theta_union_experimental_test.cpp
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <iostream>
+
+#include <catch.hpp>
+#include <tuple_union.hpp>
+
+#include <theta_union_experimental.hpp>
+
+namespace datasketches {
+
+TEST_CASE("theta_union_exeperimental") {
+ auto update_sketch1 = update_theta_sketch_experimental<>::builder().build();
+ update_sketch1.update(1);
+ update_sketch1.update(2);
+
+ auto update_sketch2 = update_theta_sketch_experimental<>::builder().build();
+ update_sketch2.update(1);
+ update_sketch2.update(3);
+
+ auto u = theta_union_experimental<>::builder().build();
+ u.update(update_sketch1);
+ u.update(update_sketch2);
+ auto r = u.get_result();
+}
+
+} /* namespace datasketches */
diff --git a/tuple/test/tuple_a_not_b_test.cpp b/tuple/test/tuple_a_not_b_test.cpp
new file mode 100644
index 0000000..1c56102
--- /dev/null
+++ b/tuple/test/tuple_a_not_b_test.cpp
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <iostream>
+
+#include <catch.hpp>
+#include <tuple_a_not_b.hpp>
+#include <theta_sketch_experimental.hpp>
+
+namespace datasketches {
+
+TEST_CASE("tuple a-not-b: empty", "[tuple_a_not_b]") {
+ auto a = update_tuple_sketch<float>::builder().build();
+ auto b = update_tuple_sketch<float>::builder().build();
+ tuple_a_not_b<float> a_not_b;
+ auto result = a_not_b.compute(a, b);
+ REQUIRE(result.get_num_retained() == 0);
+ REQUIRE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("tuple a-not-b: non empty no retained keys", "[tuple_a_not_b]") {
+ auto a = update_tuple_sketch<float>::builder().build();
+ a.update(1, 1);
+ auto b = update_tuple_sketch<float>::builder().set_p(0.001).build();
+ tuple_a_not_b<float> a_not_b;
+
+ // B is still empty
+ auto result = a_not_b.compute(a, b);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_num_retained() == 1);
+ REQUIRE(result.get_theta() == Approx(1).margin(1e-10));
+ REQUIRE(result.get_estimate() == 1.0);
+
+ // B is not empty in estimation mode and no entries
+ b.update(1, 1);
+ REQUIRE(b.get_num_retained() == 0);
+
+ result = a_not_b.compute(a, b);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_num_retained() == 0);
+ REQUIRE(result.get_theta() == Approx(0.001).margin(1e-10));
+ REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("tuple a-not-b: exact mode half overlap", "[tuple_a_not_b]") {
+ auto a = update_tuple_sketch<float>::builder().build();
+ int value = 0;
+ for (int i = 0; i < 1000; i++) a.update(value++, 1);
+
+ auto b = update_tuple_sketch<float>::builder().build();
+ value = 500;
+ for (int i = 0; i < 1000; i++) b.update(value++, 1);
+
+ tuple_a_not_b<float> a_not_b;
+
+ // unordered inputs, ordered result
+ auto result = a_not_b.compute(a, b);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.is_ordered());
+ REQUIRE(result.get_estimate() == 500.0);
+
+ // unordered inputs, unordered result
+ result = a_not_b.compute(a, b, false);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE_FALSE(result.is_ordered());
+ REQUIRE(result.get_estimate() == 500.0);
+
+ // ordered inputs
+ result = a_not_b.compute(a.compact(), b.compact());
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.is_ordered());
+ REQUIRE(result.get_estimate() == 500.0);
+
+ // A is ordered, so the result is ordered regardless
+ result = a_not_b.compute(a.compact(), b, false);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.is_ordered());
+ REQUIRE(result.get_estimate() == 500.0);
+}
+
+// needed until promotion of experimental to replace existing theta sketch
+using update_theta_sketch = update_theta_sketch_experimental<>;
+
+TEST_CASE("mixed a-not-b: exact mode half overlap", "[tuple_a_not_b]") {
+ auto a = update_tuple_sketch<float>::builder().build();
+ int value = 0;
+ for (int i = 0; i < 1000; i++) a.update(value++, 1);
+
+ auto b = update_theta_sketch::builder().build();
+ value = 500;
+ for (int i = 0; i < 1000; i++) b.update(value++);
+
+ tuple_a_not_b<float> a_not_b;
+
+ // unordered inputs, ordered result
+ auto result = a_not_b.compute(a, compact_tuple_sketch<float>(b, 1, false));
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.is_ordered());
+ REQUIRE(result.get_estimate() == 500.0);
+
+ // unordered inputs, unordered result
+ result = a_not_b.compute(a, compact_tuple_sketch<float>(b, 1, false), false);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE_FALSE(result.is_ordered());
+ REQUIRE(result.get_estimate() == 500.0);
+
+ // ordered inputs
+ result = a_not_b.compute(a.compact(), compact_tuple_sketch<float>(b.compact(), 1));
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.is_ordered());
+ REQUIRE(result.get_estimate() == 500.0);
+
+ // A is ordered, so the result is ordered regardless
+ result = a_not_b.compute(a.compact(), compact_tuple_sketch<float>(b, 1, false), false);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.is_ordered());
+ REQUIRE(result.get_estimate() == 500.0);
+}
+
+TEST_CASE("tuple a-not-b: exact mode disjoint", "[tuple_a_not_b]") {
+ auto a = update_tuple_sketch<float>::builder().build();
+ int value = 0;
+ for (int i = 0; i < 1000; i++) a.update(value++, 1);
+
+ auto b = update_tuple_sketch<float>::builder().build();
+ for (int i = 0; i < 1000; i++) b.update(value++, 1);
+
+ tuple_a_not_b<float> a_not_b;
+
+ // unordered inputs
+ auto result = a_not_b.compute(a, b);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 1000.0);
+
+ // ordered inputs
+ result = a_not_b.compute(a.compact(), b.compact());
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 1000.0);
+}
+
+TEST_CASE("tuple a-not-b: exact mode full overlap", "[tuple_a_not_b]") {
+ auto sketch = update_tuple_sketch<float>::builder().build();
+ int value = 0;
+ for (int i = 0; i < 1000; i++) sketch.update(value++, 1);
+
+ tuple_a_not_b<float> a_not_b;
+
+ // unordered inputs
+ auto result = a_not_b.compute(sketch, sketch);
+ REQUIRE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+
+ // ordered inputs
+ result = a_not_b.compute(sketch.compact(), sketch.compact());
+ REQUIRE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("tuple a-not-b: estimation mode half overlap", "[tuple_a_not_b]") {
+ auto a = update_tuple_sketch<float>::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; i++) a.update(value++, 1);
+
+ auto b = update_tuple_sketch<float>::builder().build();
+ value = 5000;
+ for (int i = 0; i < 10000; i++) b.update(value++, 1);
+
+ tuple_a_not_b<float> a_not_b;
+
+ // unordered inputs
+ auto result = a_not_b.compute(a, b);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.02));
+
+ // ordered inputs
+ result = a_not_b.compute(a.compact(), b.compact());
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.02));
+}
+
+TEST_CASE("tuple a-not-b: estimation mode disjoint", "[tuple_a_not_b]") {
+ auto a = update_tuple_sketch<float>::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; i++) a.update(value++, 1);
+
+ auto b = update_tuple_sketch<float>::builder().build();
+ for (int i = 0; i < 10000; i++) b.update(value++, 1);
+
+ tuple_a_not_b<float> a_not_b;
+
+ // unordered inputs
+ auto result = a_not_b.compute(a, b);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(10000).margin(10000 * 0.02));
+
+ // ordered inputs
+ result = a_not_b.compute(a.compact(), b.compact());
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(10000).margin(10000 * 0.02));
+}
+
+TEST_CASE("tuple a-not-b: estimation mode full overlap", "[tuple_a_not_b]") {
+ auto sketch = update_tuple_sketch<float>::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; i++) sketch.update(value++, 1);
+
+ tuple_a_not_b<float> a_not_b;
+
+ // unordered inputs
+ auto result = a_not_b.compute(sketch, sketch);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+
+ // ordered inputs
+ result = a_not_b.compute(sketch.compact(), sketch.compact());
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("tuple a-not-b: seed mismatch", "[tuple_a_not_b]") {
+ auto sketch = update_tuple_sketch<float>::builder().build();
+ sketch.update(1, 1); // non-empty should not be ignored
+ tuple_a_not_b<float> a_not_b(123);
+ REQUIRE_THROWS_AS(a_not_b.compute(sketch, sketch), std::invalid_argument);
+}
+
+TEST_CASE("tuple a-not-b: issue #152", "[tuple_a_not_b]") {
+ auto a = update_tuple_sketch<float>::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; i++) a.update(value++, 1);
+
+ auto b = update_tuple_sketch<float>::builder().build();
+ value = 5000;
+ for (int i = 0; i < 25000; i++) b.update(value++, 1);
+
+ tuple_a_not_b<float> a_not_b;
+
+ // unordered inputs
+ auto result = a_not_b.compute(a, b);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.03));
+
+ // ordered inputs
+ result = a_not_b.compute(a.compact(), b.compact());
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.03));
+}
+
+} /* namespace datasketches */
diff --git a/tuple/test/tuple_intersection_test.cpp b/tuple/test/tuple_intersection_test.cpp
new file mode 100644
index 0000000..9796cb3
--- /dev/null
+++ b/tuple/test/tuple_intersection_test.cpp
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <iostream>
+
+#include <catch.hpp>
+#include <tuple_intersection.hpp>
+#include <theta_sketch_experimental.hpp>
+
+namespace datasketches {
+
+template<typename Summary>
+struct subtracting_intersection_policy {
+ void operator()(Summary& summary, const Summary& other) const {
+ summary -= other;
+ }
+};
+
+using tuple_intersection_float = tuple_intersection<float, subtracting_intersection_policy<float>>;
+
+TEST_CASE("tuple intersection: invalid", "[tuple_intersection]") {
+ tuple_intersection_float intersection;
+ REQUIRE_FALSE(intersection.has_result());
+ REQUIRE_THROWS_AS(intersection.get_result(), std::invalid_argument);
+}
+
+TEST_CASE("tuple intersection: empty", "[tuple_intersection]") {
+ auto sketch = update_tuple_sketch<float>::builder().build();
+ tuple_intersection_float intersection;
+ intersection.update(sketch);
+ auto result = intersection.get_result();
+ REQUIRE(result.get_num_retained() == 0);
+ REQUIRE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+
+ intersection.update(sketch);
+ result = intersection.get_result();
+ REQUIRE(result.get_num_retained() == 0);
+ REQUIRE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("tuple intersection: non empty no retained keys", "[tuple_intersection]") {
+ auto sketch = update_tuple_sketch<float>::builder().set_p(0.001).build();
+ sketch.update(1, 1);
+ tuple_intersection_float intersection;
+ intersection.update(sketch);
+ auto result = intersection.get_result();
+ REQUIRE(result.get_num_retained() == 0);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_theta() == Approx(0.001).margin(1e-10));
+ REQUIRE(result.get_estimate() == 0.0);
+
+ intersection.update(sketch);
+ result = intersection.get_result();
+ REQUIRE(result.get_num_retained() == 0);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_theta() == Approx(0.001).margin(1e-10));
+ REQUIRE(result.get_estimate() == 0.0);
+}
+
+TEST_CASE("tuple intersection: exact mode half overlap", "[tuple_intersection]") {
+ auto sketch1 = update_tuple_sketch<float>::builder().build();
+ int value = 0;
+ for (int i = 0; i < 1000; i++) sketch1.update(value++, 1);
+
+ auto sketch2 = update_tuple_sketch<float>::builder().build();
+ value = 500;
+ for (int i = 0; i < 1000; i++) sketch2.update(value++, 1);
+
+ { // unordered
+ tuple_intersection_float intersection;
+ intersection.update(sketch1);
+ intersection.update(sketch2);
+ auto result = intersection.get_result();
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 500.0);
+ }
+ { // ordered
+ tuple_intersection_float intersection;
+ intersection.update(sketch1.compact());
+ intersection.update(sketch2.compact());
+ auto result = intersection.get_result();
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 500.0);
+ }
+}
+
+TEST_CASE("tuple intersection: exact mode disjoint", "[tuple_intersection]") {
+ auto sketch1 = update_tuple_sketch<float>::builder().build();
+ int value = 0;
+ for (int i = 0; i < 1000; i++) sketch1.update(value++, 1);
+
+ auto sketch2 = update_tuple_sketch<float>::builder().build();
+ for (int i = 0; i < 1000; i++) sketch2.update(value++, 1);
+
+ { // unordered
+ tuple_intersection_float intersection;
+ intersection.update(sketch1);
+ intersection.update(sketch2);
+ auto result = intersection.get_result();
+ REQUIRE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+ }
+ { // ordered
+ tuple_intersection_float intersection;
+ intersection.update(sketch1.compact());
+ intersection.update(sketch2.compact());
+ auto result = intersection.get_result();
+ REQUIRE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+ }
+}
+
+// needed until promotion of experimental to replace existing theta sketch
+using update_theta_sketch = update_theta_sketch_experimental<>;
+
+TEST_CASE("mixed intersection: exact mode half overlap", "[tuple_intersection]") {
+ auto sketch1 = update_tuple_sketch<float>::builder().build();
+ int value = 0;
+ for (int i = 0; i < 1000; i++) sketch1.update(value++, 1);
+
+ auto sketch2 = update_theta_sketch::builder().build();
+ value = 500;
+ for (int i = 0; i < 1000; i++) sketch2.update(value++);
+
+ { // unordered
+ tuple_intersection_float intersection;
+ intersection.update(sketch1);
+ intersection.update(compact_tuple_sketch<float>(sketch2, 1, false));
+ auto result = intersection.get_result();
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 500.0);
+ }
+ { // ordered
+ tuple_intersection_float intersection;
+ intersection.update(sketch1.compact());
+ intersection.update(compact_tuple_sketch<float>(sketch2.compact(), 1));
+ auto result = intersection.get_result();
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE_FALSE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 500.0);
+ }
+}
+
+TEST_CASE("tuple intersection: estimation mode half overlap", "[tuple_intersection]") {
+ auto sketch1 = update_tuple_sketch<float>::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; i++) sketch1.update(value++, 1);
+
+ auto sketch2 = update_tuple_sketch<float>::builder().build();
+ value = 5000;
+ for (int i = 0; i < 10000; i++) sketch2.update(value++, 1);
+
+ { // unordered
+ tuple_intersection_float intersection;
+ intersection.update(sketch1);
+ intersection.update(sketch2);
+ auto result = intersection.get_result();
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.02));
+ }
+ { // ordered
+ tuple_intersection_float intersection;
+ intersection.update(sketch1.compact());
+ intersection.update(sketch2.compact());
+ auto result = intersection.get_result();
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(5000).margin(5000 * 0.02));
+ }
+}
+
+TEST_CASE("tuple intersection: estimation mode disjoint", "[tuple_intersection]") {
+ auto sketch1 = update_tuple_sketch<float>::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; i++) sketch1.update(value++, 1);
+
+ auto sketch2 = update_tuple_sketch<float>::builder().build();
+ for (int i = 0; i < 10000; i++) sketch2.update(value++, 1);
+
+ { // unordered
+ tuple_intersection_float intersection;
+ intersection.update(sketch1);
+ intersection.update(sketch2);
+ auto result = intersection.get_result();
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+ }
+ { // ordered
+ tuple_intersection_float intersection;
+ intersection.update(sketch1.compact());
+ intersection.update(sketch2.compact());
+ auto result = intersection.get_result();
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0.0);
+ }
+}
+
+TEST_CASE("tuple intersection: seed mismatch", "[tuple_intersection]") {
+ auto sketch = update_tuple_sketch<float>::builder().build();
+ sketch.update(1, 1); // non-empty should not be ignored
+ tuple_intersection_float intersection(123);
+ REQUIRE_THROWS_AS(intersection.update(sketch), std::invalid_argument);
+}
+
+} /* namespace datasketches */
diff --git a/tuple/test/tuple_jaccard_similarity_test.cpp b/tuple/test/tuple_jaccard_similarity_test.cpp
new file mode 100644
index 0000000..9545593
--- /dev/null
+++ b/tuple/test/tuple_jaccard_similarity_test.cpp
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <iostream>
+
+#include <catch.hpp>
+#include <jaccard_similarity.hpp>
+
+namespace datasketches {
+
+using tuple_jaccard_similarity_float = tuple_jaccard_similarity<float, default_union_policy<float>>;
+
+TEST_CASE("tuple jaccard: empty", "[tuple_sketch]") {
+ auto sk_a = update_tuple_sketch<float>::builder().build();
+ auto sk_b = update_tuple_sketch<float>::builder().build();
+
+ // update sketches
+ auto jc = tuple_jaccard_similarity_float::jaccard(sk_a, sk_b);
+ REQUIRE(jc == std::array<double, 3>{1, 1, 1});
+
+ // compact sketches
+ jc = tuple_jaccard_similarity_float::jaccard(sk_a.compact(), sk_b.compact());
+ REQUIRE(jc == std::array<double, 3>{1, 1, 1});
+
+ REQUIRE(tuple_jaccard_similarity_float::exactly_equal(sk_a, sk_b));
+}
+
+TEST_CASE("tuple jaccard: same sketch exact mode", "[tuple_sketch]") {
+ auto sk = update_tuple_sketch<float>::builder().build();
+ for (int i = 0; i < 1000; ++i) sk.update(i, 1);
+
+ // update sketch
+ auto jc = tuple_jaccard_similarity_float::jaccard(sk, sk);
+ REQUIRE(jc == std::array<double, 3>{1, 1, 1});
+
+ // compact sketch
+ jc = tuple_jaccard_similarity_float::jaccard(sk.compact(), sk.compact());
+ REQUIRE(jc == std::array<double, 3>{1, 1, 1});
+
+ REQUIRE(tuple_jaccard_similarity_float::exactly_equal(sk, sk));
+}
+
+TEST_CASE("tuple jaccard: full overlap exact mode", "[tuple_sketch]") {
+ auto sk_a = update_tuple_sketch<float>::builder().build();
+ auto sk_b = update_tuple_sketch<float>::builder().build();
+ for (int i = 0; i < 1000; ++i) {
+ sk_a.update(i, 1);
+ sk_b.update(i, 1);
+ }
+
+ // update sketches
+ auto jc = tuple_jaccard_similarity_float::jaccard(sk_a, sk_b);
+ REQUIRE(jc == std::array<double, 3>{1, 1, 1});
+
+ // compact sketches
+ jc = tuple_jaccard_similarity_float::jaccard(sk_a.compact(), sk_b.compact());
+ REQUIRE(jc == std::array<double, 3>{1, 1, 1});
+
+ REQUIRE(tuple_jaccard_similarity_float::exactly_equal(sk_a, sk_b));
+ REQUIRE(tuple_jaccard_similarity_float::exactly_equal(sk_a.compact(), sk_b));
+ REQUIRE(tuple_jaccard_similarity_float::exactly_equal(sk_a, sk_b.compact()));
+ REQUIRE(tuple_jaccard_similarity_float::exactly_equal(sk_a.compact(), sk_b.compact()));
+}
+
+TEST_CASE("tuple jaccard: disjoint exact mode", "[tuple_sketch]") {
+ auto sk_a = update_tuple_sketch<float>::builder().build();
+ auto sk_b = update_tuple_sketch<float>::builder().build();
+ for (int i = 0; i < 1000; ++i) {
+ sk_a.update(i, 1);
+ sk_b.update(i + 1000, 1);
+ }
+
+ // update sketches
+ auto jc = tuple_jaccard_similarity_float::jaccard(sk_a, sk_b);
+ REQUIRE(jc == std::array<double, 3>{0, 0, 0});
+
+ // compact sketches
+ jc = tuple_jaccard_similarity_float::jaccard(sk_a.compact(), sk_b.compact());
+ REQUIRE(jc == std::array<double, 3>{0, 0, 0});
+}
+
+} /* namespace datasketches */
diff --git a/tuple/test/tuple_sketch_allocation_test.cpp b/tuple/test/tuple_sketch_allocation_test.cpp
new file mode 100644
index 0000000..d87c06e
--- /dev/null
+++ b/tuple/test/tuple_sketch_allocation_test.cpp
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <iostream>
+
+#include <catch.hpp>
+#include <tuple_sketch.hpp>
+#include <test_allocator.hpp>
+#include <test_type.hpp>
+
+namespace datasketches {
+
+static const bool ALLOCATOR_TEST_DEBUG = false;
+
+struct test_type_replace_policy {
+ test_type create() const { return test_type(0); }
+ void update(test_type& summary, const test_type& update) const {
+ if (ALLOCATOR_TEST_DEBUG) std::cerr << "policy::update lvalue begin" << std::endl;
+ summary = update;
+ if (ALLOCATOR_TEST_DEBUG) std::cerr << "policy::update lvalue end" << std::endl;
+ }
+ void update(test_type& summary, test_type&& update) const {
+ if (ALLOCATOR_TEST_DEBUG) std::cerr << "policy::update rvalue begin" << std::endl;
+ summary = std::move(update);
+ if (ALLOCATOR_TEST_DEBUG) std::cerr << "policy::update rvalue end" << std::endl;
+ }
+};
+
+using update_tuple_sketch_test =
+ update_tuple_sketch<test_type, test_type, test_type_replace_policy, test_allocator<test_type>>;
+using compact_tuple_sketch_test =
+ compact_tuple_sketch<test_type, test_allocator<test_type>>;
+
+TEST_CASE("tuple sketch with test allocator: estimation mode", "[tuple_sketch]") {
+ test_allocator_total_bytes = 0;
+ test_allocator_net_allocations = 0;
+ {
+ auto update_sketch = update_tuple_sketch_test::builder().build();
+ for (int i = 0; i < 10000; ++i) update_sketch.update(i, 1);
+ for (int i = 0; i < 10000; ++i) update_sketch.update(i, 2);
+ REQUIRE(!update_sketch.is_empty());
+ REQUIRE(update_sketch.is_estimation_mode());
+ unsigned count = 0;
+ for (const auto& entry: update_sketch) {
+ REQUIRE(entry.second.get_value() == 2);
+ ++count;
+ }
+ REQUIRE(count == update_sketch.get_num_retained());
+
+ update_sketch.trim();
+ REQUIRE(update_sketch.get_num_retained() == (1 << update_sketch.get_lg_k()));
+
+ auto compact_sketch = update_sketch.compact();
+ REQUIRE(!compact_sketch.is_empty());
+ REQUIRE(compact_sketch.is_estimation_mode());
+ count = 0;
+ for (const auto& entry: compact_sketch) {
+ REQUIRE(entry.second.get_value() == 2);
+ ++count;
+ }
+ REQUIRE(count == update_sketch.get_num_retained());
+
+ auto bytes = compact_sketch.serialize(0, test_type_serde());
+ auto deserialized_sketch = compact_tuple_sketch_test::deserialize(bytes.data(), bytes.size(), DEFAULT_SEED, test_type_serde());
+ REQUIRE(deserialized_sketch.get_estimate() == compact_sketch.get_estimate());
+
+ // update sketch copy
+ if (ALLOCATOR_TEST_DEBUG) std::cout << update_sketch.to_string();
+ update_tuple_sketch_test update_sketch_copy(update_sketch);
+ update_sketch_copy = update_sketch;
+ // update sketch move
+ update_tuple_sketch_test update_sketch_moved(std::move(update_sketch_copy));
+ update_sketch_moved = std::move(update_sketch);
+
+ // compact sketch copy
+ compact_tuple_sketch_test compact_sketch_copy(compact_sketch);
+ compact_sketch_copy = compact_sketch;
+ // compact sketch move
+ compact_tuple_sketch_test compact_sketch_moved(std::move(compact_sketch_copy));
+ compact_sketch_moved = std::move(compact_sketch);
+ }
+ REQUIRE(test_allocator_total_bytes == 0);
+ REQUIRE(test_allocator_net_allocations == 0);
+}
+
+} /* namespace datasketches */
diff --git a/tuple/test/tuple_sketch_test.cpp b/tuple/test/tuple_sketch_test.cpp
new file mode 100644
index 0000000..ec5d959
--- /dev/null
+++ b/tuple/test/tuple_sketch_test.cpp
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <iostream>
+#include <tuple>
+
+namespace datasketches {
+
+using three_doubles = std::tuple<double, double, double>;
+
+// this is needed for a test below, but should be defined here
+std::ostream& operator<<(std::ostream& os, const three_doubles& tuple) {
+ os << std::get<0>(tuple) << ", " << std::get<1>(tuple) << ", " << std::get<2>(tuple);
+ return os;
+}
+
+}
+
+#include <catch.hpp>
+#include <tuple_sketch.hpp>
+//#include <test_type.hpp>
+
+namespace datasketches {
+
+TEST_CASE("tuple sketch float: builder", "[tuple_sketch]") {
+ auto builder = update_tuple_sketch<float>::builder();
+ builder.set_lg_k(10).set_p(0.5).set_resize_factor(theta_constants::resize_factor::X2).set_seed(123);
+ auto sketch = builder.build();
+ REQUIRE(sketch.get_lg_k() == 10);
+ REQUIRE(sketch.get_theta() == 0.5);
+ REQUIRE(sketch.get_rf() == theta_constants::resize_factor::X2);
+ REQUIRE(sketch.get_seed_hash() == compute_seed_hash(123));
+}
+
+TEST_CASE("tuple sketch float: empty", "[tuple_sketch]") {
+ auto update_sketch = update_tuple_sketch<float>::builder().build();
+ std::cout << "sizeof(update_tuple_sketch<float>)=" << sizeof(update_sketch) << std::endl;
+ REQUIRE(update_sketch.is_empty());
+ REQUIRE(!update_sketch.is_estimation_mode());
+ REQUIRE(update_sketch.get_estimate() == 0);
+ REQUIRE(update_sketch.get_lower_bound(1) == 0);
+ REQUIRE(update_sketch.get_upper_bound(1) == 0);
+ REQUIRE(update_sketch.get_theta() == 1);
+ REQUIRE(update_sketch.get_num_retained() == 0);
+ REQUIRE(!update_sketch.is_ordered());
+
+ auto compact_sketch = update_sketch.compact();
+ std::cout << "sizeof(compact_tuple_sketch<float>)=" << sizeof(compact_sketch) << std::endl;
+ REQUIRE(compact_sketch.is_empty());
+ REQUIRE(!compact_sketch.is_estimation_mode());
+ REQUIRE(compact_sketch.get_estimate() == 0);
+ REQUIRE(compact_sketch.get_lower_bound(1) == 0);
+ REQUIRE(compact_sketch.get_upper_bound(1) == 0);
+ REQUIRE(compact_sketch.get_theta() == 1);
+ REQUIRE(compact_sketch.get_num_retained() == 0);
+ REQUIRE(compact_sketch.is_ordered());
+}
+
+TEST_CASE("tuple sketch float: exact mode", "[tuple_sketch]") {
+ auto update_sketch = update_tuple_sketch<float>::builder().build();
+ update_sketch.update(1, 1);
+ update_sketch.update(2, 2);
+ update_sketch.update(1, 1);
+// std::cout << update_sketch.to_string(true);
+ REQUIRE(!update_sketch.is_empty());
+ REQUIRE(!update_sketch.is_estimation_mode());
+ REQUIRE(update_sketch.get_estimate() == 2);
+ REQUIRE(update_sketch.get_lower_bound(1) == 2);
+ REQUIRE(update_sketch.get_upper_bound(1) == 2);
+ REQUIRE(update_sketch.get_theta() == 1);
+ REQUIRE(update_sketch.get_num_retained() == 2);
+ REQUIRE(!update_sketch.is_ordered());
+ int count = 0;
+ for (const auto& entry: update_sketch) {
+ REQUIRE(entry.second == 2);
+ ++count;
+ }
+ REQUIRE(count == 2);
+
+ auto compact_sketch = update_sketch.compact();
+// std::cout << compact_sketch.to_string(true);
+ REQUIRE(!compact_sketch.is_empty());
+ REQUIRE(!compact_sketch.is_estimation_mode());
+ REQUIRE(compact_sketch.get_estimate() == 2);
+ REQUIRE(compact_sketch.get_lower_bound(1) == 2);
+ REQUIRE(compact_sketch.get_upper_bound(1) == 2);
+ REQUIRE(compact_sketch.get_theta() == 1);
+ REQUIRE(compact_sketch.get_num_retained() == 2);
+ REQUIRE(compact_sketch.is_ordered());
+ count = 0;
+ for (const auto& entry: compact_sketch) {
+ REQUIRE(entry.second == 2);
+ ++count;
+ }
+ REQUIRE(count == 2);
+
+ { // stream
+ std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+ compact_sketch.serialize(s);
+ auto deserialized_sketch = compact_tuple_sketch<float>::deserialize(s);
+ REQUIRE(!deserialized_sketch.is_empty());
+ REQUIRE(!deserialized_sketch.is_estimation_mode());
+ REQUIRE(deserialized_sketch.get_estimate() == 2);
+ REQUIRE(deserialized_sketch.get_lower_bound(1) == 2);
+ REQUIRE(deserialized_sketch.get_upper_bound(1) == 2);
+ REQUIRE(deserialized_sketch.get_theta() == 1);
+ REQUIRE(deserialized_sketch.get_num_retained() == 2);
+ REQUIRE(deserialized_sketch.is_ordered());
+// std::cout << "deserialized sketch:" << std::endl;
+// std::cout << deserialized_sketch.to_string(true);
+ }
+ { // bytes
+ auto bytes = compact_sketch.serialize();
+ auto deserialized_sketch = compact_tuple_sketch<float>::deserialize(bytes.data(), bytes.size());
+ REQUIRE(!deserialized_sketch.is_empty());
+ REQUIRE(!deserialized_sketch.is_estimation_mode());
+ REQUIRE(deserialized_sketch.get_estimate() == 2);
+ REQUIRE(deserialized_sketch.get_lower_bound(1) == 2);
+ REQUIRE(deserialized_sketch.get_upper_bound(1) == 2);
+ REQUIRE(deserialized_sketch.get_theta() == 1);
+ REQUIRE(deserialized_sketch.get_num_retained() == 2);
+ REQUIRE(deserialized_sketch.is_ordered());
+// std::cout << deserialized_sketch.to_string(true);
+ }
+ // mixed
+ {
+ auto bytes = compact_sketch.serialize();
+ std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
+ s.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+ auto deserialized_sketch = compact_tuple_sketch<float>::deserialize(s);
+ auto it = deserialized_sketch.begin();
+ for (const auto& entry: compact_sketch) {
+ REQUIRE(entry.first == (*it).first);
+ REQUIRE(entry.second == (*it).second);
+ ++it;
+ }
+ }
+}
+
+template<typename T>
+class max_value_policy {
+public:
+ max_value_policy(const T& initial_value): initial_value(initial_value) {}
+ T create() const { return initial_value; }
+ void update(T& summary, const T& update) const { summary = std::max(summary, update); }
+private:
+ T initial_value;
+};
+
+using max_float_update_tuple_sketch = update_tuple_sketch<float, float, max_value_policy<float>>;
+
+TEST_CASE("tuple sketch: float, custom policy", "[tuple_sketch]") {
+ auto update_sketch = max_float_update_tuple_sketch::builder(max_value_policy<float>(5)).build();
+ update_sketch.update(1, 1);
+ update_sketch.update(1, 2);
+ update_sketch.update(2, 10);
+ update_sketch.update(3, 3);
+ update_sketch.update(3, 7);
+// std::cout << update_sketch.to_string(true);
+ int count = 0;
+ float sum = 0;
+ for (const auto& entry: update_sketch) {
+ sum += entry.second;
+ ++count;
+ }
+ REQUIRE(count == 3);
+ REQUIRE(sum == 22); // 5 + 10 + 7
+}
+
+struct three_doubles_update_policy {
+ std::tuple<double, double, double> create() const {
+ return std::tuple<double, double, double>(0, 0, 0);
+ }
+ void update(std::tuple<double, double, double>& summary, const std::tuple<double, double, double>& update) const {
+ std::get<0>(summary) += std::get<0>(update);
+ std::get<1>(summary) += std::get<1>(update);
+ std::get<2>(summary) += std::get<2>(update);
+ }
+};
+
+TEST_CASE("tuple sketch: tuple of doubles", "[tuple_sketch]") {
+ using three_doubles_update_tuple_sketch = update_tuple_sketch<three_doubles, three_doubles, three_doubles_update_policy>;
+ auto update_sketch = three_doubles_update_tuple_sketch::builder().build();
+ update_sketch.update(1, three_doubles(1, 2, 3));
+// std::cout << update_sketch.to_string(true);
+ const auto& entry = *update_sketch.begin();
+ REQUIRE(std::get<0>(entry.second) == 1.0);
+ REQUIRE(std::get<1>(entry.second) == 2.0);
+ REQUIRE(std::get<2>(entry.second) == 3.0);
+
+ auto compact_sketch = update_sketch.compact();
+// std::cout << compact_sketch.to_string(true);
+ REQUIRE(compact_sketch.get_num_retained() == 1);
+}
+
+TEST_CASE("tuple sketch: float, update with different types of keys", "[tuple_sketch]") {
+ auto sketch = update_tuple_sketch<float>::builder().build();
+
+ sketch.update(static_cast<uint64_t>(1), 1);
+ REQUIRE(sketch.get_num_retained() == 1);
+
+ sketch.update(static_cast<int64_t>(1), 1);
+ REQUIRE(sketch.get_num_retained() == 1);
+
+ sketch.update(static_cast<uint32_t>(1), 1);
+ REQUIRE(sketch.get_num_retained() == 1);
+
+ sketch.update(static_cast<int32_t>(1), 1);
+ REQUIRE(sketch.get_num_retained() == 1);
+
+ sketch.update(static_cast<uint16_t>(1), 1);
+ REQUIRE(sketch.get_num_retained() == 1);
+
+ sketch.update(static_cast<int16_t>(1), 1);
+ REQUIRE(sketch.get_num_retained() == 1);
+
+ sketch.update(static_cast<uint8_t>(1), 1);
+ REQUIRE(sketch.get_num_retained() == 1);
+
+ sketch.update(static_cast<int8_t>(1), 1);
+ REQUIRE(sketch.get_num_retained() == 1);
+
+ sketch.update(1.0, 1);
+ REQUIRE(sketch.get_num_retained() == 2);
+
+ sketch.update(static_cast<float>(1), 1);
+ REQUIRE(sketch.get_num_retained() == 2);
+
+ sketch.update("a", 1);
+ REQUIRE(sketch.get_num_retained() == 3);
+}
+
+} /* namespace datasketches */
diff --git a/tuple/test/tuple_union_test.cpp b/tuple/test/tuple_union_test.cpp
new file mode 100644
index 0000000..281b37c
--- /dev/null
+++ b/tuple/test/tuple_union_test.cpp
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <iostream>
+
+#include <catch.hpp>
+#include <tuple_union.hpp>
+#include <theta_sketch_experimental.hpp>
+
+namespace datasketches {
+
+TEST_CASE("tuple_union float: empty", "[tuple union]") {
+ auto update_sketch = update_tuple_sketch<float>::builder().build();
+ auto u = tuple_union<float>::builder().build();
+ u.update(update_sketch);
+ auto result = u.get_result();
+// std::cout << result.to_string(true);
+ REQUIRE(result.is_empty());
+ REQUIRE(result.get_num_retained() == 0);
+ REQUIRE(!result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0);
+}
+
+// needed until promotion of experimental to replace existing theta sketch
+using update_theta_sketch = update_theta_sketch_experimental<>;
+
+TEST_CASE("tupe_union float: empty theta sketch", "[tuple union]") {
+ auto update_sketch = update_theta_sketch::builder().build();
+
+ auto u = tuple_union<float>::builder().build();
+ u.update(compact_tuple_sketch<float>(update_sketch, 0));
+ auto result = u.get_result();
+// std::cout << result.to_string(true);
+ REQUIRE(result.is_empty());
+ REQUIRE(result.get_num_retained() == 0);
+ REQUIRE(!result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0);
+}
+
+TEST_CASE("tuple_union float: non-empty no retained entries", "[tuple union]") {
+ auto update_sketch = update_tuple_sketch<float>::builder().set_p(0.001).build();
+// std::cout << update_sketch.to_string();
+ update_sketch.update(1, 1);
+ REQUIRE(!update_sketch.is_empty());
+ REQUIRE(update_sketch.get_num_retained() == 0);
+ auto u = tuple_union<float>::builder().build();
+ u.update(update_sketch);
+ auto result = u.get_result();
+// std::cout << result.to_string(true);
+ REQUIRE(!result.is_empty());
+ REQUIRE(result.get_num_retained() == 0);
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 0);
+ REQUIRE(result.get_theta() == Approx(0.001).margin(1e-10));
+}
+
+TEST_CASE("tuple_union float: simple case", "[tuple union]") {
+ auto update_sketch1 = update_tuple_sketch<float>::builder().build();
+ update_sketch1.update(1, 1);
+ update_sketch1.update(2, 1);
+
+ auto update_sketch2 = update_tuple_sketch<float>::builder().build();
+ update_sketch2.update(1, 1);
+ update_sketch2.update(3, 1);
+
+ auto u = tuple_union<float>::builder().build();
+ u.update(update_sketch1);
+ u.update(update_sketch2);
+ auto result = u.get_result();
+ REQUIRE(result.get_num_retained() == 3);
+}
+
+TEST_CASE("tuple_union float: exact mode half overlap", "[tuple union]") {
+ auto update_sketch1 = update_tuple_sketch<float>::builder().build();
+ int value = 0;
+ for (int i = 0; i < 1000; ++i) update_sketch1.update(value++, 1);
+
+ auto update_sketch2 = update_tuple_sketch<float>::builder().build();
+ value = 500;
+ for (int i = 0; i < 1000; ++i) update_sketch2.update(value++, 1);
+
+ { // unordered
+ auto u = tuple_union<float>::builder().build();
+ u.update(update_sketch1);
+ u.update(update_sketch2);
+ auto result = u.get_result();
+ REQUIRE(!result.is_empty());
+ REQUIRE(!result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(1500).margin(1500 * 0.01));
+ }
+ { // ordered
+ auto u = tuple_union<float>::builder().build();
+ u.update(update_sketch1.compact());
+ u.update(update_sketch2.compact());
+ auto result = u.get_result();
+ REQUIRE(!result.is_empty());
+ REQUIRE(!result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(1500).margin(1500 * 0.01));
+ }
+}
+
+TEST_CASE("tuple_union float: estimation mode half overlap", "[tuple union]") {
+ auto update_sketch1 = update_tuple_sketch<float>::builder().build();
+ int value = 0;
+ for (int i = 0; i < 10000; ++i) update_sketch1.update(value++, 1);
+
+ auto update_sketch2 = update_tuple_sketch<float>::builder().build();
+ value = 5000;
+ for (int i = 0; i < 10000; ++i) update_sketch2.update(value++, 1);
+
+ { // unordered
+ auto u = tuple_union<float>::builder().build();
+ u.update(update_sketch1);
+ u.update(update_sketch2);
+ auto result = u.get_result();
+ REQUIRE(!result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(15000).margin(15000 * 0.01));
+ }
+ { // ordered
+ auto u = tuple_union<float>::builder().build();
+ u.update(update_sketch1.compact());
+ u.update(update_sketch2.compact());
+ auto result = u.get_result();
+ REQUIRE(!result.is_empty());
+ REQUIRE(result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == Approx(15000).margin(15000 * 0.01));
+ }
+}
+
+TEST_CASE("tuple_union float: seed mismatch", "[tuple union]") {
+ auto update_sketch = update_tuple_sketch<float>::builder().build();
+ update_sketch.update(1, 1); // non-empty should not be ignored
+
+ auto u = tuple_union<float>::builder().set_seed(123).build();
+ REQUIRE_THROWS_AS(u.update(update_sketch), std::invalid_argument);
+}
+
+TEST_CASE("tuple_union float: full overlap with theta sketch", "[tuple union]") {
+ auto u = tuple_union<float>::builder().build();
+
+ // tuple update
+ auto update_tuple = update_tuple_sketch<float>::builder().build();
+ for (unsigned i = 0; i < 10; ++i) update_tuple.update(i, 1);
+ u.update(update_tuple);
+
+ // tuple compact
+ auto compact_tuple = update_tuple.compact();
+ u.update(compact_tuple);
+
+ // theta update
+ auto update_theta = update_theta_sketch::builder().build();
+ for (unsigned i = 0; i < 10; ++i) update_theta.update(i);
+ u.update(compact_tuple_sketch<float>(update_theta, 1));
+
+ // theta compact
+ auto compact_theta = update_theta.compact();
+ u.update(compact_tuple_sketch<float>(compact_theta, 1));
+
+ auto result = u.get_result();
+// std::cout << result.to_string(true);
+ REQUIRE_FALSE(result.is_empty());
+ REQUIRE(result.get_num_retained() == 10);
+ REQUIRE(!result.is_estimation_mode());
+ REQUIRE(result.get_estimate() == 10);
+ for (const auto& entry: result) {
+ REQUIRE(entry.second == 4);
+ }
+}
+
+} /* namespace datasketches */
