copy from prototype repo
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..27fbfa1
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,424 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright 2017, Yahoo! Inc.
+ Licensed under the terms of the Apache License 2.0.
+ See LICENSE file at the project root for terms. -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>svd</groupId>
+ <artifactId>svd</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <!-- Project Information -->
+ <name>${project.groupId}:${project.artifactId}</name>
+ <description>Sketches for matrix approximation.</description>
+ <url>https://datasketches.github.io/</url>
+ <inceptionYear>2017</inceptionYear>
+
+ <licenses>
+ <license>
+ <name>Apache License, Version 2.0</name>
+ <url>https://www.apache.org/licenses/LICENSE-2.0</url>
+ </license>
+ </licenses>
+
+ <organization>
+ <name>Yahoo! Inc.</name>
+ <url>https://www.yahoo.com</url>
+ </organization>
+
+ <developers>
+ <developer>
+ <name>Jon Malkin</name>
+ <roles>
+ <role>lead developer</role>
+ </roles>
+ <url>https://github.com/jmalkin</url>
+ </developer>
+ </developers>
+
+ <contributors>
+ <contributor>
+ <url>https://github.com/DataSketches/sketches-core/graphs/contributors</url>
+ </contributor>
+ </contributors>
+ <!-- End Project Information -->
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.build.resourceEncoding>UTF-8</project.build.resourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ <argLine>-Xmx1024m -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8</argLine>
+ </properties>
+
+ <!-- Environment Settings -->
+ <issueManagement>
+ <system>GitHub Issues</system>
+ <url>https://github.com/DataSketches/sketches-core/issues</url>
+ </issueManagement>
+
+ <ciManagement>
+ <system>travis</system>
+ <url>https://travis-ci.org/DataSketches/sketches-core</url>
+ </ciManagement>
+
+ <mailingLists>
+ <mailingList>
+ <name>sketches-user</name>
+ <archive>https://groups.google.com/forum/#!forum/sketches-user</archive>
+ <subscribe>mailto:sketches-user%2Bsubscribe@googlegroups.com</subscribe>
+ <unsubscribe>mailto:sketches-user%2Bunsubscribe@googlegroups.com</unsubscribe>
+ <post>mailto:sketches-user@googlegroups.com</post>
+ </mailingList>
+ </mailingLists>
+
+ <scm>
+ <connection>scm:git:ssh://git@github.com/DataSketches/sketches-core.git</connection>
+ <developerConnection>scm:git:ssh://git@github.com/DataSketches/sketches-core.git</developerConnection>
+ <url>https://github.com/DataSketches/sketches-core.git</url>
+ <tag>HEAD</tag>
+ </scm>
+
+ <prerequisites>
+ <maven>3.0.4</maven>
+ </prerequisites>
+
+ <repositories>
+ <repository>
+ <id>jcenter</id>
+ <name>bintray</name>
+ <url>https://jcenter.bintray.com</url>
+ </repository>
+ </repositories>
+
+ <distributionManagement>
+ <repository>
+ <id>sonatype-nexus-staging</id>
+ <url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
+ </repository>
+ </distributionManagement>
+
+ <!-- This is optional -->
+ <profiles>
+ <profile>
+ <id>strict</id>
+ <build>
+ <pluginManagement>
+ <plugins>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>${maven.compiler.source}</source>
+ <target>${maven.compiler.target}</target>
+ <compilerId>javac-with-errorprone</compilerId>
+ <forceJavacCompilerUse>true</forceJavacCompilerUse>
+ </configuration>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-compiler-javac-errorprone</artifactId>
+ <version>2.8.2</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+
+ </plugins>
+ </pluginManagement>
+ </build>
+ </profile>
+ </profiles>
+ <!-- End of Environment Settings -->
+
+ <dependencies>
+ <dependency>
+ <groupId>org.ojalgo</groupId>
+ <artifactId>ojalgo</artifactId>
+ <version>43.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.yahoo.datasketches</groupId>
+ <artifactId>memory</artifactId>
+ <version>0.10.2</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.pig</groupId>
+ <artifactId>pig</artifactId>
+ <version>0.16.0</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>2.8.1</version>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>asm</groupId>
+ <artifactId>asm</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Test Scope -->
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <version>6.11</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- run locally for now
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>findbugs</artifactId>
+ <version>3.0.1</version>
+ <scope>test</scope>
+ </dependency>
+ -->
+ </dependencies>
+
+ <build>
+ <plugins>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <relocations>
+ <relocation>
+ <pattern>com.yahoo.memory</pattern>
+ <shadedPattern>shaded.com.yahoo.memory</shadedPattern>
+ </relocation>
+ </relocations>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>with-shaded-memory</shadedClassifierName>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <!--
+ <show>private</show>
+ -->
+ <stylesheetfile>src/main/javadoc/stylesheet.css</stylesheetfile>
+ <docfilessubdirs>true</docfilessubdirs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>attach-javadocs</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>attach-sources</id>
+ <goals>
+ <goal>jar-no-fork</goal>
+ <goal>test-jar-no-fork</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.sonatype.plugins</groupId>
+ <artifactId>nexus-staging-maven-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <serverId>sonatype-nexus-staging</serverId>
+ <nexusUrl>https://oss.sonatype.org/</nexusUrl>
+ <autoReleaseAfterClose>false</autoReleaseAfterClose>
+ </configuration>
+ </plugin>
+
+ <!-- Code coverage plugin, generates coverage report to target/site/jacoco/
+ To skip coverage generation add -Djacoco.skip=true -->
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>prepare-agent</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>report</id>
+ <phase>test</phase>
+ <goals>
+ <goal>report</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-gpg-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>sign-artifacts</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>sign</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+
+ <pluginManagement>
+ <plugins>
+ <!-- ORG.APACHE.MAVEN -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>3.0.0</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-clean-plugin</artifactId>
+ <version>3.0.0</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.6.1</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>3.0.1</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <version>2.8.2</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-gpg-plugin</artifactId>
+ <version>1.6</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-help-plugin</artifactId>
+ <version>2.2</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-install-plugin</artifactId>
+ <version>2.5.2</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>2.10.4</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-release-plugin</artifactId>
+ <version>2.5.3</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>3.0.2</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.0.0</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>3.0.1</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.20</version>
+ </plugin>
+
+ <!-- OTHER -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.6.0</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>license-maven-plugin</artifactId>
+ <version>1.13</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <version>0.7.9</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.sonatype.plugins</groupId>
+ <artifactId>nexus-staging-maven-plugin</artifactId>
+ <version>1.6.8</version>
+ </plugin>
+
+ <plugin>
+ <groupId>com.versioneye</groupId>
+ <artifactId>versioneye-maven-plugin</artifactId>
+ <version>3.11.4</version>
+ </plugin>
+
+ </plugins>
+ </pluginManagement>
+ </build>
+</project>
diff --git a/src/main/java/com/yahoo/sketches/MatrixFamily.java b/src/main/java/com/yahoo/sketches/MatrixFamily.java
new file mode 100644
index 0000000..07b4674
--- /dev/null
+++ b/src/main/java/com/yahoo/sketches/MatrixFamily.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2017, Yahoo, Inc.
+ * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
+ */
+
+package com.yahoo.sketches;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Defines the various families of sketch and set operation classes. A family defines a set of
+ * classes that share fundamental algorithms and behaviors. The classes within a family may
+ * still differ by how they are stored and accessed. For example, internally there may be separate
+ * classes for algorithms that operate on the Java heap and off-heap.
+ * Not all of these families have parallel forms on and off-heap but are included for completeness.
+ *
+ * <p>Family IDs start at 128 to allow separation from sketches-core for as long as possible without
+ * inducing a mutual dependency between packages.</p>
+ *
+ * @author Lee Rhodes
+ * @author Jon Malkin
+ */
+public enum MatrixFamily {
+ /**
+ * The Frequent Directions sketch is used for approximate Singular Value Decomposition (SVD) of a
+ * matrix.
+ */
+ MATRIX(128, "FrequentDirections", 2, 3),
+ FREQUENTDIRECTIONS(129, "FrequentDirections", 2, 4);
+
+
+ private static final Map<Integer, MatrixFamily> lookupID = new HashMap<>();
+ private static final Map<String, MatrixFamily> lookupFamName = new HashMap<>();
+ private int id_;
+ private String famName_;
+ private int minPreLongs_;
+ private int maxPreLongs_;
+
+ static {
+ for (MatrixFamily f : values()) {
+ lookupID.put(f.getID(), f);
+ lookupFamName.put(f.getFamilyName().toUpperCase(), f);
+ }
+ }
+
+ MatrixFamily(final int id, final String famName, final int minPreLongs, final int maxPreLongs) {
+ id_ = id;
+ famName_ = famName.toUpperCase();
+ minPreLongs_ = minPreLongs;
+ maxPreLongs_ = maxPreLongs;
+ }
+
+ /**
+ * Returns the byte ID for this family
+ * @return the byte ID for this family
+ */
+ public int getID() {
+ return id_;
+ }
+
+ /**
+ *
+ * @param id the given id, a value ≥ 128.
+ */
+ public void checkFamilyID(final int id) {
+ if (id != id_) {
+ throw new SketchesArgumentException(
+ "Possible Corruption: This Family " + this.toString()
+ + " does not match the ID of the given Family: " + idToFamily(id).toString());
+ }
+ }
+
+ /**
+ * Returns the name for this family
+ * @return the name for this family
+ */
+ public String getFamilyName() {
+ return famName_;
+ }
+
+ /**
+ * Returns the minimum preamble size for this family in longs
+ * @return the minimum preamble size for this family in longs
+ */
+ public int getMinPreLongs() {
+ return minPreLongs_;
+ }
+
+ /**
+ * Returns the maximum preamble size for this family in longs
+ * @return the maximum preamble size for this family in longs
+ */
+ public int getMaxPreLongs() {
+ return maxPreLongs_;
+ }
+
+ @Override
+ public String toString() {
+ return famName_;
+ }
+
+ /**
+ * Returns the Family given the ID
+ * @param id the given ID
+ * @return the Family given the ID
+ */
+ public static MatrixFamily idToFamily(final int id) {
+ final MatrixFamily f = lookupID.get(id);
+ if (f == null) {
+ throw new SketchesArgumentException("Possible Corruption: Illegal Family ID: " + id);
+ }
+ return f;
+ }
+
+ /**
+ * Returns the Family given the family name
+ * @param famName the family name
+ * @return the Family given the family name
+ */
+ public static MatrixFamily stringToFamily(final String famName) {
+ final MatrixFamily f = lookupFamName.get(famName.toUpperCase());
+ if (f == null) {
+ throw new SketchesArgumentException("Possible Corruption: Illegal Family Name: " + famName);
+ }
+ return f;
+ }
+
+ /**
+ * Returns the Family given one of the recognized class objects on one of the Families
+ * @param obj a recognized Family class object
+ * @return the Family given one of the recognized class objects on one of the Families
+ */
+ public static MatrixFamily objectToFamily(final Object obj) {
+ final String sname = obj.getClass().getSimpleName().toUpperCase();
+ for (MatrixFamily f : values()) {
+ if (sname.contains(f.toString())) {
+ return f;
+ }
+ }
+ throw new SketchesArgumentException("Possible Corruption: Unknown object");
+ }
+}
diff --git a/src/main/java/com/yahoo/sketches/SketchesArgumentException.java b/src/main/java/com/yahoo/sketches/SketchesArgumentException.java
new file mode 100644
index 0000000..7683696
--- /dev/null
+++ b/src/main/java/com/yahoo/sketches/SketchesArgumentException.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2015-16, Yahoo! Inc.
+ * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
+ */
+
+package com.yahoo.sketches;
+
+/**
+ * Illegal Arguments Exception class for the library
+ *
+ * @author Lee Rhodes
+ */
+public class SketchesArgumentException extends RuntimeException {
+ private static final long serialVersionUID = 1L;
+
+ //other constructors to be added as needed.
+
+ /**
+ * Constructs a new runtime exception with the specified detail message. The cause is not
+ * initialized, and may subsequently be initialized by a call to
+ * Throwable.initCause(java.lang.Throwable).
+ *
+ * @param message the detail message. The detail message is saved for later retrieval by the
+ * Throwable.getMessage() method.
+ */
+ public SketchesArgumentException(final String message) {
+ super(message);
+ }
+}
diff --git a/src/main/java/com/yahoo/sketches/decomposition/FrequentDirections.java b/src/main/java/com/yahoo/sketches/decomposition/FrequentDirections.java
new file mode 100644
index 0000000..e1583d0
--- /dev/null
+++ b/src/main/java/com/yahoo/sketches/decomposition/FrequentDirections.java
@@ -0,0 +1,578 @@
+package com.yahoo.sketches.decomposition;
+
+import static com.yahoo.memory.UnsafeUtil.LS;
+import static com.yahoo.sketches.decomposition.PreambleUtil.EMPTY_FLAG_MASK;
+import static com.yahoo.sketches.decomposition.PreambleUtil.extractFamilyID;
+import static com.yahoo.sketches.decomposition.PreambleUtil.extractFlags;
+import static com.yahoo.sketches.decomposition.PreambleUtil.extractK;
+import static com.yahoo.sketches.decomposition.PreambleUtil.extractN;
+import static com.yahoo.sketches.decomposition.PreambleUtil.extractNumColumns;
+import static com.yahoo.sketches.decomposition.PreambleUtil.extractNumRows;
+import static com.yahoo.sketches.decomposition.PreambleUtil.extractSVAdjustment;
+import static com.yahoo.sketches.decomposition.PreambleUtil.extractSerVer;
+import static com.yahoo.sketches.decomposition.PreambleUtil.getAndCheckPreLongs;
+import static com.yahoo.sketches.decomposition.PreambleUtil.insertFamilyID;
+import static com.yahoo.sketches.decomposition.PreambleUtil.insertFlags;
+import static com.yahoo.sketches.decomposition.PreambleUtil.insertK;
+import static com.yahoo.sketches.decomposition.PreambleUtil.insertN;
+import static com.yahoo.sketches.decomposition.PreambleUtil.insertNumColumns;
+import static com.yahoo.sketches.decomposition.PreambleUtil.insertNumRows;
+import static com.yahoo.sketches.decomposition.PreambleUtil.insertPreLongs;
+import static com.yahoo.sketches.decomposition.PreambleUtil.insertSVAdjustment;
+import static com.yahoo.sketches.decomposition.PreambleUtil.insertSerVer;
+
+import org.ojalgo.array.Array1D;
+import org.ojalgo.matrix.decomposition.SingularValue;
+import org.ojalgo.matrix.store.MatrixStore;
+import org.ojalgo.matrix.store.PrimitiveDenseStore;
+import org.ojalgo.matrix.store.SparseStore;
+
+import com.yahoo.memory.Memory;
+import com.yahoo.memory.WritableMemory;
+import com.yahoo.sketches.MatrixFamily;
+import com.yahoo.sketches.SketchesArgumentException;
+import com.yahoo.sketches.matrix.Matrix;
+import com.yahoo.sketches.matrix.MatrixBuilder;
+
+/**
+ * This class implements the Frequent Directions algorithm proposed by Edo Liberty in "Simple and
+ * Deterministic Matrix Sketches," KDD 2013. The sketch provides an approximation to the singular
+ * value decomposition of a matrix with deterministic error bounds on the error between the
+ * approximation and the optimal rank-k matrix decomposition.
+ *
+ * @author Jon Malkin
+ */
+public final class FrequentDirections {
+ private final int k_;
+ private final int l_;
+ private final int d_;
+ private long n_;
+
+ private double svAdjustment_;
+
+ private PrimitiveDenseStore B_;
+ transient private int nextZeroRow_;
+
+ transient private final double[] sv_; // pre-allocated to fetch singular values
+ transient private final SparseStore<Double> S_; // to hold singular value matrix
+
+ /**
+ * Creates a new instance of a Frequent Directions sketch.
+ * @param k Number of dimensions (rows) in the sketch output
+ * @param d Number of dimensions per input vector (columns)
+ * @return An empty Frequent Directions sketch
+ */
+ public static FrequentDirections newInstance(final int k, final int d) {
+ return new FrequentDirections(k, d);
+ }
+
+ /**
+ * Instantiates a Frequent Directions sketch from a serialized image.
+ * @param srcMem Memory containing the serialized image of a Frequent Directions sketch
+ * @return A Frequent Directions sketch
+ */
+ public static FrequentDirections heapify(final Memory srcMem) {
+ final int preLongs = getAndCheckPreLongs(srcMem);
+ final int serVer = extractSerVer(srcMem);
+ if (serVer != PreambleUtil.SER_VER) {
+ throw new SketchesArgumentException("Invalid serialization version: " + serVer);
+ }
+
+ final int family = extractFamilyID(srcMem);
+ if (family != MatrixFamily.FREQUENTDIRECTIONS.getID()) {
+ throw new SketchesArgumentException("Possible corruption: Family id (" + family + ") "
+ + "is not a FrequentDirections sketch");
+ }
+
+ final int k = extractK(srcMem);
+ final int numRows = extractNumRows(srcMem);
+ final int d = extractNumColumns(srcMem);
+ final boolean empty = (extractFlags(srcMem) & EMPTY_FLAG_MASK) > 0;
+
+ if (empty) {
+ return new FrequentDirections(k, d);
+ }
+
+ final long offsetBytes = preLongs * Long.BYTES;
+ final long mtxBytes = srcMem.getCapacity() - offsetBytes;
+ final Matrix B = Matrix.heapify(srcMem.region(offsetBytes, mtxBytes), MatrixBuilder.Algo.OJALGO);
+ assert B != null;
+
+ final FrequentDirections fd
+ = new FrequentDirections(k, d, (PrimitiveDenseStore) B.getRawObject());
+ fd.n_ = extractN(srcMem);
+ fd.nextZeroRow_ = numRows;
+ fd.svAdjustment_ = extractSVAdjustment(srcMem);
+
+ return fd;
+ }
+
+ private FrequentDirections(final int k, final int d) {
+ this(k, d, null);
+ }
+
+ private FrequentDirections(final int k, final int d, final PrimitiveDenseStore B) {
+ if (k < 1) {
+ throw new IllegalArgumentException("Number of projected dimensions must be at least 1");
+ }
+ if (d < 1) {
+ throw new IllegalArgumentException("Number of feature dimensions must be at least 1");
+ }
+
+ k_ = k;
+ l_ = 2 * k;
+ d_ = d;
+
+ if (d_ < l_) {
+ throw new IllegalArgumentException("Running with d < 2k not yet supported");
+ }
+
+ svAdjustment_ = 0.0;
+
+ nextZeroRow_ = 0;
+ n_ = 0;
+
+ if (B == null) {
+ B_ = PrimitiveDenseStore.FACTORY.makeZero(l_, d_);
+ } else {
+ B_ = B;
+ }
+
+ final int svDim = Math.min(l_, d_);
+ sv_ = new double[svDim];
+ S_ = SparseStore.makePrimitive(svDim, svDim);
+ }
+
+ /**
+ * Update sketch with a dense input vector of exactly d dimensions.
+ * @param vector A dense input vector representing one row of the input matrix
+ */
+ public void update(final double[] vector) {
+ if (vector == null) {
+ return;
+ }
+
+ if (vector.length != d_) {
+ throw new IllegalArgumentException("Input vector has too few dimensions. Expected " + d_
+ + "; found " + vector.length);
+ }
+
+ if (nextZeroRow_ == l_) {
+ reduceRank();
+ }
+
+ // dense input so set all values
+ for (int i = 0; i < vector.length; ++i) {
+ B_.set(nextZeroRow_, i, vector[i]);
+ }
+
+ ++n_;
+ ++nextZeroRow_;
+ }
+
+ /**
+ * Merge a Frequent Directions sketch into the current one.
+ * @param fd A Frequent Direction sketch to be merged.
+ */
+ public void update(final FrequentDirections fd) {
+ if (fd == null || fd.nextZeroRow_ == 0) {
+ return;
+ }
+
+ if ((fd.d_ != d_) || (fd.k_ < k_)) {
+ throw new SketchesArgumentException("Incoming sketch must have same number of dimensions "
+ + "and no smaller a value of k");
+ }
+
+ for (int m = 0; m < fd.nextZeroRow_; ++m) {
+ if (nextZeroRow_ == l_) {
+ reduceRank();
+ }
+
+ final Array1D<Double> rv = fd.B_.sliceRow(m);
+ for (int i = 0; i < rv.count(); ++i) {
+ B_.set(nextZeroRow_, i, rv.get(i));
+ }
+
+ ++nextZeroRow_;
+ }
+
+ n_ += fd.n_;
+ svAdjustment_ += fd.svAdjustment_;
+ }
+
+ /**
+ * Checks if the sketch is empty, specifically whether it has processed any input data.
+ * @return True if hte sketch has not yet processed any input
+ */
+ public boolean isEmpty() {
+ return n_ == 0;
+ }
+
+ /**
+ * Returns the target number of dimensions, k, for this sketch.
+ * @return The sketch's configured k value
+ */
+ public int getK() { return k_; }
+
+ /**
+ * Returns the number of dimensions per input vector, d, for this sketch.
+ * @return The sketch's configured number of dimensions per input
+ */
+ public int getD() { return d_; }
+
+ /**
+ * Returns the total number of items this sketch has seen.
+ * @return The number of items processed by the sketch.
+ */
+ public long getN() { return n_; }
+
+ /**
+ * Returns the singular values of the sketch, adjusted for the mass subtracted off during the
+ * algorithm.
+ * @return An array of singular values.
+ */
+ public double[] getSingularValues() {
+ return getSingularValues(true);
+ }
+
+ /**
+ * Returns the singular values of the sketch, optionally adjusting for any mass subtracted off
+ * during the algorithm.
+ * @param compensative If true, adjusts for mass subtracted during the algorithm, otherwise
+ * uses raw singular values.
+ * @return As array of singular values.
+ */
+ public double[] getSingularValues(final boolean compensative) {
+ final SingularValue<Double> svd = SingularValue.make(B_);
+ svd.compute(B_);
+ svd.getSingularValues(sv_);
+
+ double medianSVSq = sv_[k_ - 1]; // (l_/2)th item, not yet squared
+ medianSVSq *= medianSVSq;
+ final double tmpSvAdj = svAdjustment_ + medianSVSq;
+ final double[] svList = new double[k_];
+
+ for (int i = 0; i < k_ - 1; ++i) {
+ final double val = sv_[i];
+ double adjSqSV = val * val - medianSVSq;
+ if (compensative) { adjSqSV += tmpSvAdj; }
+ svList[i] = adjSqSV < 0 ? 0.0 : Math.sqrt(adjSqSV);
+ }
+
+ return svList;
+ }
+
+ /**
+ * Returns an orthonormal projection Matrix that can be use to project input vectors into the
+ * k-dimensional space represented by the sketch.
+ * @return An orthonormal Matrix object
+ */
+ public Matrix getProjectionMatrix() {
+ final SingularValue<Double> svd = SingularValue.make(B_);
+ svd.compute(B_);
+ final MatrixStore<Double> m = svd.getQ2().transpose();
+
+ // not super efficient...
+ final Matrix result = Matrix.builder().build(k_, d_);
+ for (int i = 0; i < k_ - 1; ++i) { // last SV is 0
+ result.setRow(i, m.sliceRow(i).toRawCopy1D());
+ }
+
+ return result;
+ }
+
+ /**
+ * Calls <tt>getResult(true, false)</tt>
+ * @return A Matrix representing the data in this sketch
+ */
+ public Matrix getResult() {
+ return getResult(true, false);
+ }
+
+ /**
+ * Returns a Matrix with the sketch's estimate of the SVD of the input data.
+ * @param compress If true, force compression down to no more than k vectors
+ * @param compensative If true, applies adjustment to singular values based on the cumulative
+ * weight subtracted off
+ * @return A Matrix representing the data in this sketch
+ */
+ public Matrix getResult(final boolean compress, final boolean compensative) {
+ if (isEmpty()) {
+ return null;
+ }
+
+ if (compress && nextZeroRow_ > k_) {
+ reduceRank();
+ }
+
+ final PrimitiveDenseStore result;
+
+ if (compensative) {
+ // in the event we just called reduceRank(), the high rows are already zeroed out so no need
+ // to do so again
+ final SingularValue<Double> svd = SingularValue.make(B_);
+ svd.compute(B_);
+ svd.getSingularValues(sv_);
+
+ for (int i = 0; i < k_ - 1; ++i) {
+ final double val = sv_[i];
+ final double adjSV = Math.sqrt(val * val + svAdjustment_);
+ S_.set(i, i, adjSV);
+ }
+ for (int i = k_ - 1; i < S_.countColumns(); ++i) {
+ S_.set(i, i, 0.0);
+ }
+
+ //result = PrimitiveDenseStore.FACTORY.makeZero(l_, d_);
+ result = PrimitiveDenseStore.FACTORY.makeZero(nextZeroRow_, d_);
+ S_.multiply(svd.getQ2().transpose(), result);
+ } else {
+ result = PrimitiveDenseStore.FACTORY.makeZero(nextZeroRow_, d_);
+ for (int i = 0; i < nextZeroRow_; ++i) {
+ int j = 0;
+ for (double d : B_.sliceRow(i)) {
+ result.set(i, j++, d);
+ }
+ }
+ }
+
+ return Matrix.wrap(result);
+ }
+
+ /**
+ * Resets the sketch to its virgin state.
+ */
+ public void reset() {
+ n_ = 0;
+ nextZeroRow_ = 0;
+ }
+
+ /**
+ * Returns a serialized representation of the sketch. Equivalent to calling <tt>toByteArray
+ * (true)</tt>.
+ * <p>Note: May modify sketch state. If the sketch would store more than k rows, applies SVD to
+ * compress the sketch to examply k rows.</p>
+ * @return A serialized representation of the sketch.
+ */
+ public byte[] toByteArray() {
+ return toByteArray(true);
+ }
+
+ /**
+ * Returns a serialized representation of the sketch.
+ * <p>Note: If compress is true, will modify sketch state if the sketch would store more than k
+ * rows by applying SVD to compress the sketch to examply k rows.</p>
+ * @param compress If true, compresses teh sketch to no more than k rows.
+ * @return A serialized representation of the sketch.
+ */
+ public byte[] toByteArray(final boolean compress) {
+ final boolean empty = isEmpty();
+ final int serVer = 1;
+ final int familyId = MatrixFamily.FREQUENTDIRECTIONS.getID();
+
+ final Matrix wrapB = Matrix.wrap(B_);
+
+ // project down to k rows to serialize, chasing the 2GB byte[] limit
+ if (compress && nextZeroRow_ > k_) {
+ reduceRank();
+ }
+
+ final int preLongs = empty
+ ? MatrixFamily.FREQUENTDIRECTIONS.getMinPreLongs()
+ : MatrixFamily.FREQUENTDIRECTIONS.getMaxPreLongs();
+
+ final int mtxBytes = empty ? 0 : wrapB.getCompactSizeBytes(nextZeroRow_, d_);
+ final int outBytes = (preLongs * Long.BYTES) + mtxBytes;
+
+ final byte[] outArr = new byte[outBytes];
+ final WritableMemory memOut = WritableMemory.wrap(outArr);
+ final Object memObj = memOut.getArray();
+ final long memAddr = memOut.getCumulativeOffset(0L);
+
+ insertPreLongs(memObj, memAddr, preLongs);
+ insertSerVer(memObj, memAddr, serVer);
+ insertFamilyID(memObj, memAddr, familyId);
+ insertFlags(memObj, memAddr, (empty ? EMPTY_FLAG_MASK : 0));
+ insertK(memObj, memAddr, k_);
+ insertNumRows(memObj, memAddr, nextZeroRow_);
+ insertNumColumns(memObj, memAddr, d_);
+
+ if (empty) {
+ return outArr;
+ }
+
+ insertN(memObj, memAddr, n_);
+ insertSVAdjustment(memObj, memAddr, svAdjustment_);
+
+ memOut.putByteArray(preLongs * Long.BYTES,
+ wrapB.toCompactByteArray(nextZeroRow_, d_), 0, mtxBytes);
+
+ return outArr;
+ }
+
+ @Override
+ public String toString() {
+ return toString(false, false, false);
+ }
+
+ /**
+ * Returns a human-readable summary of the sketch and, optionally, prints the raw data.
+ * @param printMatrix If true, prints sketch's data matrix
+ * @return A String representation of the sketch.
+ */
+ public String toString(final boolean printMatrix) {
+ return toString(printMatrix, false, false);
+ }
+
+ /**
+ * Returns a human-readable summary of the sketch, optionally printing either the filled
+ * or complete sketch matrix, and also optionally adjusting the singular values based on the
+ * total weight subtacted during the algorithm.
+ * @param printMatrix If true, prints the sketch's data matrix
+ * @param fullMatrix If true, prints all rows; if false, only non-empty rows
+ * @param applyCompensation If true, prints adjusted singular values
+ * @return A String representation of the sketch.
+ */
+ public String toString(final boolean printMatrix, final boolean fullMatrix,
+ final boolean applyCompensation) {
+ final StringBuilder sb = new StringBuilder();
+
+ final String thisSimpleName = this.getClass().getSimpleName();
+
+ sb.append(LS);
+ sb.append("### ").append(thisSimpleName).append(" INFO: ").append(LS);
+ if (applyCompensation) {
+ sb.append("Applying compensative adjustments to matrix values").append(LS);
+ }
+ sb.append(" k : ").append(k_).append(LS);
+ sb.append(" d : ").append(d_).append(LS);
+ sb.append(" l : ").append(l_).append(LS);
+ sb.append(" n : ").append(n_).append(LS);
+ sb.append(" numRows : ").append(nextZeroRow_).append(LS);
+ sb.append(" SV adjustment: ").append(svAdjustment_).append(LS);
+
+ if (!printMatrix) {
+ return sb.toString();
+ }
+
+ sb.append(" Singular Vals: ")
+ .append(applyCompensation ? "(adjusted)" : "(unadjusted)").append(LS);
+ final double[] sv = getSingularValues(applyCompensation);
+ for (int i = 0; i < Math.min(k_, n_); ++i) {
+ if (sv[i] > 0.0) {
+ double val = sv[i];
+ if (val > 0.0 && applyCompensation) {
+ val = Math.sqrt(val * val + svAdjustment_);
+ }
+
+ sb.append(" \t").append(i).append(":\t").append(val).append(LS);
+ }
+ }
+
+ final Matrix mtx = Matrix.wrap(B_);
+
+ final int tmpRowDim = fullMatrix ? nextZeroRow_ : Math.min(k_, nextZeroRow_);
+ final int tmpColDim = (int) mtx.getNumColumns();
+
+ sb.append(" Matrix data :").append(LS);
+ sb.append(mtx.getClass().getName());
+ sb.append(" < ").append(tmpRowDim).append(" x ").append(tmpColDim).append(" >");
+
+ // First element
+ sb.append("\n{ { ").append(mtx.getElement(0, 0));
+
+ // Rest of the first row
+ for (int j = 1; j < tmpColDim; j++) {
+ sb.append(",\t").append(mtx.getElement(0, j));
+ }
+
+ // For each of the remaining rows
+ for (int i = 1; i < tmpRowDim; i++) {
+
+ // First column
+ sb.append(" },\n{ ").append(mtx.getElement(i, 0));
+
+ // Remaining columns
+ for (int j = 1; j < tmpColDim; j++) {
+ sb.append(",\t").append(mtx.getElement(i, j));
+ }
+ }
+
+ // Finish
+ sb.append(" } }").append(LS);
+ sb.append("### END SKETCH SUMMARY").append(LS);
+
+ return sb.toString();
+ }
+
+ int getNumRows() { return nextZeroRow_; }
+
+ // exists for testing
+ double getSvAdjustment() { return svAdjustment_; }
+
+ private void reduceRank() {
+ //++numReduce;
+
+ final SingularValue<Double> svd = SingularValue.make(B_);
+ svd.compute(B_);
+ svd.getSingularValues(sv_);
+
+ if (sv_.length >= k_) {
+ double medianSVSq = sv_[k_ - 1]; // (l_/2)th item, not yet squared
+ medianSVSq *= medianSVSq;
+ svAdjustment_ += medianSVSq; // always track, even if not using compensative mode
+ for (int i = 0; i < k_ - 1; ++i) {
+ final double val = sv_[i];
+ final double adjSqSV = val * val - medianSVSq;
+ S_.set(i, i, adjSqSV < 0 ? 0.0 : Math.sqrt(adjSqSV));
+ }
+ for (int i = k_ - 1; i < S_.countColumns(); ++i) {
+ S_.set(i, i, 0.0);
+ }
+ nextZeroRow_ = k_;
+ } else {
+ for (int i = 0; i < sv_.length; ++i) {
+ S_.set(i, i, sv_[i]);
+ }
+ for (int i = sv_.length; i < S_.countColumns(); ++i) {
+ S_.set(i, i, 0.0);
+ }
+ nextZeroRow_ = sv_.length;
+ throw new RuntimeException("Running with d < 2k not yet supported");
+ }
+
+ S_.multiply(svd.getQ2().transpose()).supplyTo(B_);
+ }
+
+ /*
+ private static double computeFrobNorm(final MatrixStore<Double> M) {
+ double sum = 0.0;
+ for (double d : M) {
+ sum += d * d;
+ }
+ return Math.sqrt(sum);
+ }
+
+ private static double computeFrobNorm(final MatrixStore<Double> M, final int k) {
+ double sum = 0.0;
+ for (int i = 0; i < k; ++i) {
+ for (double d : M.sliceRow(i)) {
+ sum += d * d;
+ }
+ }
+ return Math.sqrt(sum);
+ }
+
+ private static MatrixStore<Double> getKRows(final MatrixStore<Double> M, final int k) {
+ PrimitiveDenseStore result = PrimitiveDenseStore.FACTORY.makeZero(k, M.countColumns());
+ for (int i = 0; i < k; ++i) {
+ int j = 0;
+ for (double d : M.sliceRow(i)) {
+ result.set(i, j++, d);
+ }
+ }
+ return result;
+ }
+ */
+}
diff --git a/src/main/java/com/yahoo/sketches/decomposition/PreambleUtil.java b/src/main/java/com/yahoo/sketches/decomposition/PreambleUtil.java
new file mode 100644
index 0000000..a1d14f4
--- /dev/null
+++ b/src/main/java/com/yahoo/sketches/decomposition/PreambleUtil.java
@@ -0,0 +1,228 @@
+/*
+ * Copyright 2017, Yahoo, Inc.
+ * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
+ */
+
+package com.yahoo.sketches.decomposition;
+
+import static com.yahoo.memory.UnsafeUtil.unsafe;
+
+import com.yahoo.memory.Memory;
+import com.yahoo.sketches.MatrixFamily;
+import com.yahoo.sketches.SketchesArgumentException;
+
+/**
+ * This class defines the preamble items structure and provides basic utilities for some of the key fields.
+ *
+ * <p>
+ * The low significance bytes of this <tt>long</tt> items structure are on the right. Multi-byte
+ * integers (<tt>int</tt> and <tt>long</tt>) are stored in native byte order. All <tt>byte</tt>
+ * values are treated as unsigned.</p>
+ *
+ * <p>An empty Frequent Directions sketch requires 16 bytes. A non-empty sketch requires 32 bytes
+ * of preamble. The matrix is dense and is expected to dominate storage.</p>
+ *
+ * <pre>
+ * Long || Start Byte Adr:
+ * Adr:
+ * || 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 |
+ * 0 ||----------Sketch Size (k)----------| Flags | FamID | SerVer | Preamble_Longs |
+ *
+ * || 15 | 14 | 13 | 12 | 11 | 10 | 9 | 8 |
+ * 1 ||-----------Num. Columns------------|------Current Num Rows-------------------------|
+ *
+ * || 23 | 22 | 21 | 20 | 19 | 18 | 17 | 16 |
+ * 2 ||-----------------------------Total Records Seen (n)--------------------------------|
+ *
+ * || 31 | 30 | 29 | 28 | 27 | 26 | 25 | 24 |
+ * 3 ||------------------------------Total SV Adjustment----------------------------------|
+ * </pre>
+ *
+ * @author Jon Malkin
+ */
+public final class PreambleUtil {
+
+ /**
+ * The java line separator character as a String.
+ */
+ public static final String LS = System.getProperty("line.separator");
+
+ private PreambleUtil() {}
+
+ // ###### DO NOT MESS WITH THIS FROM HERE ...
+ // Preamble byte Addresses
+ static final int PREAMBLE_LONGS_BYTE = 0;
+ static final int SER_VER_BYTE = 1;
+ static final int FAMILY_BYTE = 2;
+ static final int FLAGS_BYTE = 3;
+ static final int K_INT = 4;
+ static final int NUM_ROWS_INT = 8;
+ static final int NUM_COLUMNS_INT = 12;
+ static final int N_LONG = 16;
+ static final int SV_ADJUSTMENT_DOUBLE = 24;
+
+ // flag bit masks
+ static final int EMPTY_FLAG_MASK = 4;
+ static final int COMPENSATIVE_FLAG_MASK = 128;
+
+ // Other constants
+ static final int SER_VER = 1;
+
+
+ /**
+ * Returns a human readable string summary of the preamble state of the given Memory.
+ * Note: other than making sure that the given Memory size is large
+ * enough for just the preamble, this does not do much value checking of the contents of the
+ * preamble as this is primarily a tool for debugging the preamble visually.
+ *
+ * @param mem the given Memory.
+ * @return the summary preamble string.
+ */
+ public static String preambleToString(final Memory mem) {
+
+ final int preLongs = getAndCheckPreLongs(mem); // make sure we can get the assumed preamble
+ final MatrixFamily family = MatrixFamily.idToFamily(extractFamilyID(mem));
+
+ final int serVer = extractSerVer(mem);
+ final int flags = extractFlags(mem);
+ final String flagsStr = Integer.toBinaryString(flags) + ", " + flags;
+ final boolean isEmpty = (flags & EMPTY_FLAG_MASK) > 0;
+
+ final int k = extractK(mem);
+ final int d = extractNumColumns(mem);
+ final int numRows = extractNumRows(mem);
+
+ long n = 0;
+ double svAdjustment = 0.0;
+ if (!isEmpty) {
+ n = extractN(mem);
+ svAdjustment = extractSVAdjustment(mem);
+ }
+
+ final StringBuilder sb = new StringBuilder();
+ sb.append(LS)
+ .append("### START ")
+ .append(family.getFamilyName().toUpperCase())
+ .append(" PREAMBLE SUMMARY").append(LS)
+ .append("Byte 0: Preamble Longs : ").append(preLongs).append(LS)
+ .append("Byte 1: Serialization Version: ").append(serVer).append(LS)
+ .append("Byte 2: Family : ").append(family.toString()).append(LS)
+ .append("Byte 3: Flags Field : ").append(flagsStr).append(LS)
+ .append(" EMPTY : ").append(isEmpty).append(LS)
+ .append("Bytes 4-7 : Sketch Size (k) : ").append(k).append(LS)
+ .append("Bytes 8-11: Num Rows : ").append(numRows).append(LS)
+ .append("Bytes 12-15: Num Dimensions : ").append(d).append(LS);
+
+ if (!isEmpty) {
+ sb.append("Bytes 16-23: Items Seen(n) : ").append(n).append(LS);
+ sb.append("Bytes 24-31: SV Adjustment : ").append(svAdjustment).append(LS);
+ }
+
+ final long numBytes = numRows * d * Double.BYTES;
+ sb.append("TOTAL Sketch Bytes : ").append(mem.getCapacity()).append(LS)
+ .append(" Preamble Bytes : ").append(preLongs << 3).append(LS)
+ .append(" Data Bytes : ").append(numBytes).append(LS)
+ .append("### END ")
+ .append(family.getFamilyName().toUpperCase())
+ .append(" PREAMBLE SUMMARY").append(LS);
+ return sb.toString();
+ }
+
+ // Extraction methods
+
+ static int extractPreLongs(final Memory mem) {
+ return mem.getInt(PREAMBLE_LONGS_BYTE) & 0xFF;
+ }
+
+ static int extractSerVer(final Memory mem) {
+ return mem.getInt(SER_VER_BYTE) & 0xFF;
+ }
+
+ static int extractFamilyID(final Memory mem) {
+ return mem.getByte(FAMILY_BYTE) & 0xFF;
+ }
+
+ static int extractFlags(final Memory mem) {
+ return mem.getByte(FLAGS_BYTE) & 0xFF;
+ }
+
+ static int extractK(final Memory mem) {
+ return mem.getInt(K_INT);
+ }
+
+ static int extractNumRows(final Memory mem) {
+ return mem.getInt(NUM_ROWS_INT);
+ }
+
+ static int extractNumColumns(final Memory mem) {
+ return mem.getInt(NUM_COLUMNS_INT);
+ }
+
+ static long extractN(final Memory mem) {
+ return mem.getLong(N_LONG);
+ }
+
+ static double extractSVAdjustment(final Memory mem) {
+ return mem.getDouble(SV_ADJUSTMENT_DOUBLE);
+ }
+
+
+ // Insertion methods
+
+ static void insertPreLongs(final Object memObj, final long memAddr, final int preLongs) {
+ unsafe.putByte(memObj, memAddr + PREAMBLE_LONGS_BYTE, (byte) preLongs);
+ }
+
+ static void insertSerVer(final Object memObj, final long memAddr, final int serVer) {
+ unsafe.putByte(memObj, memAddr + SER_VER_BYTE, (byte) serVer);
+ }
+
+ static void insertFamilyID(final Object memObj, final long memAddr, final int matrixFamId) {
+ unsafe.putByte(memObj, memAddr + FAMILY_BYTE, (byte) matrixFamId);
+ }
+
+ static void insertFlags(final Object memObj, final long memAddr, final int flags) {
+ unsafe.putByte(memObj, memAddr + FLAGS_BYTE, (byte) flags);
+ }
+
+ static void insertK(final Object memObj, final long memAddr, final int k) {
+ unsafe.putInt(memObj, memAddr + K_INT, k);
+ }
+
+ static void insertNumRows(final Object memObj, final long memAddr, final int numRows) {
+ unsafe.putInt(memObj, memAddr + NUM_ROWS_INT, numRows);
+ }
+
+ static void insertNumColumns(final Object memObj, final long memAddr, final int numColumns) {
+ unsafe.putInt(memObj, memAddr + NUM_COLUMNS_INT, numColumns);
+ }
+
+ static void insertN(final Object memObj, final long memAddr, final long n) {
+ unsafe.putLong(memObj, memAddr + N_LONG, n);
+ }
+
+ static void insertSVAdjustment(final Object memObj, final long memAddr, final double adj) {
+ unsafe.putDouble(memObj, memAddr + SV_ADJUSTMENT_DOUBLE, adj);
+ }
+
+
+ /**
+ * Checks Memory for capacity to hold the preamble and returns the extracted preLongs.
+ * @param mem the given Memory
+ * @return the extracted prelongs value.
+ */
+ static int getAndCheckPreLongs(final Memory mem) {
+ final long cap = mem.getCapacity();
+ if (cap < 8) { throwNotBigEnough(cap, 8); }
+ final int preLongs = extractPreLongs(mem);
+ final int required = Math.max(preLongs << 3, 8);
+ if (cap < required) { throwNotBigEnough(cap, required); }
+ return preLongs;
+ }
+
+ private static void throwNotBigEnough(final long cap, final int required) {
+ throw new SketchesArgumentException(
+ "Possible Corruption: Size of byte array or Memory not large enough: Size: " + cap
+ + ", Required: " + required);
+ }
+}
diff --git a/src/main/java/com/yahoo/sketches/decomposition/package-info.java b/src/main/java/com/yahoo/sketches/decomposition/package-info.java
new file mode 100644
index 0000000..b51f02d
--- /dev/null
+++ b/src/main/java/com/yahoo/sketches/decomposition/package-info.java
@@ -0,0 +1,14 @@
+/*
+ * Copyright 2017, Yahoo! Inc.
+ * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root
+ * for terms.
+ */
+
+/**
+ * <p>This package is dedicated to streaming algorithms that enable approximate matrix
+ * decompositions.</p>
+ *
+ * <p>These sketches are mergeable and can be serialized and deserialized to/from a compact
+ * form.</p>
+ */
+package com.yahoo.sketches.decomposition;
\ No newline at end of file
diff --git a/src/main/java/com/yahoo/sketches/matrix/Matrix.java b/src/main/java/com/yahoo/sketches/matrix/Matrix.java
new file mode 100644
index 0000000..e91a69b
--- /dev/null
+++ b/src/main/java/com/yahoo/sketches/matrix/Matrix.java
@@ -0,0 +1,217 @@
+/*
+ * Copyright 2017, Yahoo! Inc.
+ * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root
+ * for terms.
+ */
+
+package com.yahoo.sketches.matrix;
+
+import static com.yahoo.sketches.matrix.MatrixPreambleUtil.LS;
+
+import org.ojalgo.matrix.store.PrimitiveDenseStore;
+
+import com.yahoo.memory.Memory;
+import com.yahoo.sketches.MatrixFamily;
+import com.yahoo.sketches.SketchesArgumentException;
+
+
+/**
+ * Provides an implementation-agnostic wrapper around Matrix classes.
+ *
+ * @author Jon Malkin
+ */
+public abstract class Matrix {
+ int numRows_;
+ int numCols_;
+
+ /**
+ * Loads matrix from srcMem, assuming storage in column-major order to ensure portability.
+ * Does not necessarily encode matrix size; do not expect size checks based on passed-in
+ * parameters.
+ *
+ * @param srcMem Memory wrapping the matrix
+ * @param type Matrix implementation type to use
+ * @return The heapified matrix
+ */
+ public static Matrix heapify(final Memory srcMem, final MatrixBuilder.Algo type) {
+ switch (type) {
+ case OJALGO:
+ return MatrixImplOjAlgo.heapifyInstance(srcMem);
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Wraps an object without allocating memory. This method will throw an exception if the mtx
+ * Object is not of the same type as the implementing class's native format.
+ * @param mtx Matrix object to wrap
+ * @return A Matrix object
+ */
+ public static Matrix wrap(final Object mtx) {
+ if (mtx == null) {
+ return null;
+ } else if (mtx instanceof PrimitiveDenseStore) {
+ return MatrixImplOjAlgo.wrap((PrimitiveDenseStore) mtx);
+ }
+ else {
+ throw new SketchesArgumentException("wrap() does not currently support "
+ + mtx.getClass().toString());
+ }
+ }
+
+ /**
+ * Gets a builder to be able to create instances of Matrix objects
+ * @return a MatrixBuilder object
+ */
+ public static MatrixBuilder builder() {
+ return new MatrixBuilder();
+ }
+
+ /**
+ * Returns the raw data object backing this Matrix, as an Object. Must be cast to the
+ * appropriate type (assuming knowledge of the implementation) to be used.
+ * @return An Object pointing to the raw data backing this Matrix
+ */
+ public abstract Object getRawObject();
+
+ /**
+ * Serializes the Matrix in a custom format as a byte array
+ * @return A byte[] conttaining a serialized Matrix
+ */
+ public abstract byte[] toByteArray();
+
+ /**
+ * Serializes a sub-Matrix by storing only the first numRows and numCols rows and columns,
+ * respsectively.
+ * @param numRows Number of rows to write
+ * @param numCols Number of columns to write
+ * @return A byte[] containing the serialized sub-Matrix.
+ */
+ public abstract byte[] toCompactByteArray(int numRows, int numCols);
+
+ /**
+ * Returns a single element from the Matrix
+ * @param row Row index of target element (0-based)
+ * @param col Column index of target elemtn (0-based)
+ * @return Matrix value at (row, column)
+ */
+ public abstract double getElement(int row, int col);
+
+ /**
+ * Returns a copy of an entire row of the Matrix
+ * @param row Row index to return (0-based)
+ * @return A double[] representing the Matrix row
+ */
+ public abstract double[] getRow(int row);
+
+ /**
+ * Returns a copy of an entire column of the Matrix
+ * @param col Column index to return (0-based)
+ * @return A double[] representing the Matrix column
+ */
+ public abstract double[] getColumn(int col);
+
+ /**
+ * Sets a single element inthe Matrix
+ * @param row Row index of target element (0-based)
+ * @param col Column index of target element (0-based)
+ * @param value The value to insert into the Matrix at (row, column)
+ */
+ public abstract void setElement(int row, int col, double value);
+
+ /**
+ * Sets an entire row of the Matrix, by copying data from the input
+ * @param row Target row index (0-based)
+ * @param values Array of values to write into the Matrix
+ */
+ public abstract void setRow(int row, double[] values);
+
+ /**
+ * Sets an entire column of the Matrix, by copying data from the input
+ * @param column Target column index (0-based)
+ * @param values Array of values to write into the Matrix
+ */
+ public abstract void setColumn(int column, double[] values);
+
+ /**
+ * Gets the number of rows in the Matrix
+ * @return Configured number of rows in the Matrix
+ */
+ public long getNumRows() { return numRows_; }
+
+ /**
+ * Gets the number of columns in the Matrix
+ * @return Configured number of columns in the Matrix
+ */
+ public long getNumColumns() { return numCols_; }
+
+ /**
+ * Gets serialized size of the Matrix, in bytes.
+ * @return Number of bytes needed for a serialized Matrix
+ */
+ public int getSizeBytes() {
+ final int preBytes = MatrixFamily.MATRIX.getMinPreLongs() * Long.BYTES;
+ final int mtxBytes = (numRows_ * numCols_) * Double.BYTES;
+ return preBytes + mtxBytes;
+ }
+
+ /**
+ * Gets serialized size of the Matrix in cmpact form, in bytes.
+ * @param rows Number of rows to select for writing
+ * @param cols Number of columns to select for writing
+ * @return Number of bytes needed to serialize the first (rows, cols) of this Matrix
+ */
+ public int getCompactSizeBytes(final int rows, final int cols) {
+ final int nRows = Math.min(rows, numRows_);
+ final int nCols = Math.min(cols, numCols_);
+
+ if (nRows < 1 || nCols < 1) {
+ return MatrixFamily.MATRIX.getMinPreLongs() * Long.BYTES;
+ } else if (nRows == numRows_ && nCols == numCols_) {
+ return getSizeBytes();
+ }
+
+ final int preBytes = MatrixFamily.MATRIX.getMaxPreLongs() * Long.BYTES;
+ final int mtxBytes = (nRows * nCols) * Double.BYTES;
+ return preBytes + mtxBytes;
+ }
+
+ /**
+ * Writes information about this Matrix to a String.
+ * @return A human-readable representation of a Matrix
+ */
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder();
+
+ sb.append(" Matrix data :").append(LS);
+ sb.append(this.getClass().getName());
+ sb.append(" < ").append(numRows_).append(" x ").append(numCols_).append(" >");
+
+ // First element
+ sb.append("\n{ { ").append(getElement(0, 0));
+
+ // Rest of the first row
+ for (int j = 1; j < numCols_; j++) {
+ sb.append(",\t").append(getElement(0, j));
+ }
+
+ // For each of the remaining rows
+ for (int i = 1; i < numRows_; i++) {
+
+ // First column
+ sb.append(" },\n{ ").append(getElement(i, 0));
+
+ // Remaining columns
+ for (int j = 1; j < numCols_; j++) {
+ sb.append(",\t").append(getElement(i, j));
+ }
+ }
+
+ // Finish
+ sb.append(" } }").append(LS);
+
+ return sb.toString();
+ }
+}
diff --git a/src/main/java/com/yahoo/sketches/matrix/MatrixBuilder.java b/src/main/java/com/yahoo/sketches/matrix/MatrixBuilder.java
new file mode 100644
index 0000000..b21f063
--- /dev/null
+++ b/src/main/java/com/yahoo/sketches/matrix/MatrixBuilder.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2017, Yahoo! Inc.
+ * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root
+ * for terms.
+ */
+
+package com.yahoo.sketches.matrix;
+
+import com.yahoo.sketches.SketchesArgumentException;
+
+/**
+ * Provides a builder for Matrix objects.
+ */
+public class MatrixBuilder {
+ public enum Algo {
+ OJALGO(1, "ojAlgo"),
+ NATIVE(2, "native");
+
+ private int id_;
+ private String name_;
+
+ Algo(final int id, final String name) {
+ id_ = id;
+ name_ = name;
+ }
+
+ public int getId() { return id_; }
+
+ public String getName() { return name_; }
+
+ @Override
+ public String toString() { return name_; }
+ }
+
+ private Algo type_ = Algo.OJALGO; // default type
+
+ MatrixBuilder() {}
+
+ /**
+ * Sets the underlying type of object to use with any Matrix objects created.
+ * @param type One of the supported types
+ * @return This MatrixBuilder object
+ */
+ public MatrixBuilder setType(final Algo type) {
+ type_ = type;
+ return this;
+ }
+
+ /**
+ * Returns a value from an enum definig the type of object backing any Matrix objects created.
+ * @return An item from the Algo enum.
+ */
+ public Algo getFamily() {
+ return type_;
+ }
+
+ /**
+ * Instantiates a new, empty matrix of the target size
+ *
+ * @param numRows Number of rows in matrix
+ * @param numCols Number of columns in matrix
+ * @return An empty matrix of the requested size
+ */
+ public Matrix build(final int numRows, final int numCols) {
+ switch (type_) {
+ case OJALGO:
+ return MatrixImplOjAlgo.newInstance(numRows, numCols);
+
+ case NATIVE:
+ default:
+ throw new SketchesArgumentException("Only Algo.OJALGO is currently supported Matrix type");
+ }
+ }
+}
diff --git a/src/main/java/com/yahoo/sketches/matrix/MatrixImplOjAlgo.java b/src/main/java/com/yahoo/sketches/matrix/MatrixImplOjAlgo.java
new file mode 100644
index 0000000..590002b
--- /dev/null
+++ b/src/main/java/com/yahoo/sketches/matrix/MatrixImplOjAlgo.java
@@ -0,0 +1,217 @@
+/*
+ * Copyright 2017, Yahoo! Inc.
+ * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root
+ * for terms.
+ */
+
+package com.yahoo.sketches.matrix;
+
+import static com.yahoo.sketches.matrix.MatrixPreambleUtil.COMPACT_FLAG_MASK;
+import static com.yahoo.sketches.matrix.MatrixPreambleUtil.extractFamilyID;
+import static com.yahoo.sketches.matrix.MatrixPreambleUtil.extractFlags;
+import static com.yahoo.sketches.matrix.MatrixPreambleUtil.extractNumColumns;
+import static com.yahoo.sketches.matrix.MatrixPreambleUtil.extractNumColumnsUsed;
+import static com.yahoo.sketches.matrix.MatrixPreambleUtil.extractNumRows;
+import static com.yahoo.sketches.matrix.MatrixPreambleUtil.extractNumRowsUsed;
+import static com.yahoo.sketches.matrix.MatrixPreambleUtil.extractPreLongs;
+import static com.yahoo.sketches.matrix.MatrixPreambleUtil.extractSerVer;
+
+import org.ojalgo.matrix.store.PrimitiveDenseStore;
+
+import com.yahoo.memory.Memory;
+import com.yahoo.memory.WritableMemory;
+import com.yahoo.sketches.MatrixFamily;
+import com.yahoo.sketches.SketchesArgumentException;
+
+public final class MatrixImplOjAlgo extends Matrix {
+ private PrimitiveDenseStore mtx_;
+
+ private MatrixImplOjAlgo(final int numRows, final int numCols) {
+ mtx_ = PrimitiveDenseStore.FACTORY.makeZero(numRows, numCols);
+ numRows_ = numRows;
+ numCols_ = numCols;
+ }
+
+ private MatrixImplOjAlgo(final PrimitiveDenseStore mtx) {
+ mtx_ = mtx;
+ numRows_ = (int) mtx.countRows();
+ numCols_ = (int) mtx.countColumns();
+ }
+
+ static Matrix newInstance(final int numRows, final int numCols) {
+ return new MatrixImplOjAlgo(numRows, numCols);
+ }
+
+ static Matrix heapifyInstance(final Memory srcMem) {
+ final int minBytes = MatrixFamily.MATRIX.getMinPreLongs() * Long.BYTES;
+ final long memCapBytes = srcMem.getCapacity();
+ if (memCapBytes < minBytes) {
+ throw new SketchesArgumentException("Source Memory too small: " + memCapBytes
+ + " < " + minBytes);
+ }
+
+ final int preLongs = extractPreLongs(srcMem);
+ final int serVer = extractSerVer(srcMem);
+ final int familyID = extractFamilyID(srcMem);
+
+ if (serVer != 1) {
+ throw new SketchesArgumentException("Invalid SerVer reading srcMem. Expected 1, found: "
+ + serVer);
+ }
+ if (familyID != MatrixFamily.MATRIX.getID()) {
+ throw new SketchesArgumentException("srcMem does not point to a Matrix");
+ }
+
+ final int flags = extractFlags(srcMem);
+ final boolean isCompact = (flags & COMPACT_FLAG_MASK) > 0;
+
+ int nRows = extractNumRows(srcMem);
+ int nCols = extractNumColumns(srcMem);
+
+ final MatrixImplOjAlgo matrix = new MatrixImplOjAlgo(nRows, nCols);
+ if (isCompact) {
+ nRows = extractNumRowsUsed(srcMem);
+ nCols = extractNumColumnsUsed(srcMem);
+ }
+
+ int memOffset = preLongs * Long.BYTES;
+ for (int c = 0; c < nCols; ++c) {
+ for (int r = 0; r < nRows; ++r) {
+ matrix.mtx_.set(r, c, srcMem.getDouble(memOffset));
+ memOffset += Double.BYTES;
+ }
+ }
+
+ return matrix;
+ }
+
+ static Matrix wrap(final PrimitiveDenseStore mtx) {
+ return new MatrixImplOjAlgo(mtx);
+ }
+
+ public Object getRawObject() {
+ return mtx_;
+ }
+
+ @Override
+ public byte[] toByteArray() {
+ final int preLongs = 2;
+ final long numElements = mtx_.count();
+ assert numElements == mtx_.countColumns() * mtx_.countRows();
+
+ final int outBytes = (int) ((preLongs * Long.BYTES) + (numElements * Double.BYTES));
+ final byte[] outByteArr = new byte[outBytes];
+ final WritableMemory memOut = WritableMemory.wrap(outByteArr);
+ final Object memObj = memOut.getArray();
+ final long memAddr = memOut.getCumulativeOffset(0L);
+
+ MatrixPreambleUtil.insertPreLongs(memObj, memAddr, preLongs);
+ MatrixPreambleUtil.insertSerVer(memObj, memAddr, MatrixPreambleUtil.SER_VER);
+ MatrixPreambleUtil.insertFamilyID(memObj, memAddr, MatrixFamily.MATRIX.getID());
+ MatrixPreambleUtil.insertFlags(memObj, memAddr, 0);
+ MatrixPreambleUtil.insertNumRows(memObj, memAddr, (int) mtx_.countRows());
+ MatrixPreambleUtil.insertNumColumns(memObj, memAddr, (int) mtx_.countColumns());
+ memOut.putDoubleArray(preLongs << 3, mtx_.data, 0, (int) numElements);
+
+ return outByteArr;
+ }
+
+ @Override
+ public byte[] toCompactByteArray(final int numRows, final int numCols) {
+ // TODO: row/col limit checks
+
+ final int preLongs = 3;
+
+ // for non-compact we can do an array copy, so save as non-compact if using the entire matrix
+ final long numElements = numRows * numCols;
+ final boolean isCompact = numElements < mtx_.count();
+ if (!isCompact) {
+ return toByteArray();
+ }
+
+ assert numElements < mtx_.count();
+
+ //final boolean isEmpty = (numRows == 0) || (numColumns == 0);
+ //final int flags = COMPACT_FLAG_MASK | (isEmpty ? EMPTY_FLAG_MASK : 0);
+
+ final int outBytes = (int) ((preLongs * Long.BYTES) + (numElements * Double.BYTES));
+ final byte[] outByteArr = new byte[outBytes];
+ final WritableMemory memOut = WritableMemory.wrap(outByteArr);
+ final Object memObj = memOut.getArray();
+ final long memAddr = memOut.getCumulativeOffset(0L);
+
+ MatrixPreambleUtil.insertPreLongs(memObj, memAddr, preLongs);
+ MatrixPreambleUtil.insertSerVer(memObj, memAddr, MatrixPreambleUtil.SER_VER);
+ MatrixPreambleUtil.insertFamilyID(memObj, memAddr, MatrixFamily.MATRIX.getID());
+ MatrixPreambleUtil.insertFlags(memObj, memAddr, COMPACT_FLAG_MASK);
+ MatrixPreambleUtil.insertNumRows(memObj, memAddr, (int) mtx_.countRows());
+ MatrixPreambleUtil.insertNumColumns(memObj, memAddr, (int) mtx_.countColumns());
+ MatrixPreambleUtil.insertNumRowsUsed(memObj, memAddr, numRows);
+ MatrixPreambleUtil.insertNumColumnsUsed(memObj, memAddr, numCols);
+
+ // write elements in column-major order
+ long offsetBytes = preLongs * Long.BYTES;
+ for (int c = 0; c < numCols; ++c) {
+ for (int r = 0; r < numRows; ++r) {
+ memOut.putDouble(offsetBytes, mtx_.get(r, c));
+ offsetBytes += Double.BYTES;
+ }
+ }
+
+ return outByteArr;
+ }
+
+ @Override
+ public double getElement(final int row, final int col) {
+ return mtx_.get(row, col);
+ }
+
+ @Override
+ public double[] getRow(final int row) {
+ final int cols = (int) mtx_.countColumns();
+ final double[] result = new double[cols];
+ for (int c = 0; c < cols; ++c) {
+ result[c] = mtx_.get(row, c);
+ }
+ return result;
+ }
+
+ @Override
+ public double[] getColumn(final int col) {
+ final int rows = (int) mtx_.countRows();
+ final double[] result = new double[rows];
+ for (int r = 0; r < rows; ++r) {
+ result[r] = mtx_.get(r, col);
+ }
+ return result;
+ }
+
+ @Override
+ public void setElement(final int row, final int col, final double value) {
+ mtx_.set(row, col, value);
+ }
+
+ @Override
+ public void setRow(final int row, final double[] values) {
+ if (values.length != mtx_.countColumns()) {
+ throw new SketchesArgumentException("Invalid number of elements for row. Expected "
+ + mtx_.countColumns() + ", found " + values.length);
+ }
+
+ for (int i = 0; i < mtx_.countColumns(); ++i) {
+ mtx_.set(row, i, values[i]);
+ }
+ }
+
+ @Override
+ public void setColumn(final int column, final double[] values) {
+ if (values.length != mtx_.countRows()) {
+ throw new SketchesArgumentException("Invalid number of elements for column. Expected "
+ + mtx_.countRows() + ", found " + values.length);
+ }
+
+ for (int i = 0; i < mtx_.countRows(); ++i) {
+ mtx_.set(i, column, values[i]);
+ }
+ }
+}
diff --git a/src/main/java/com/yahoo/sketches/matrix/MatrixPreambleUtil.java b/src/main/java/com/yahoo/sketches/matrix/MatrixPreambleUtil.java
new file mode 100644
index 0000000..9a5af74
--- /dev/null
+++ b/src/main/java/com/yahoo/sketches/matrix/MatrixPreambleUtil.java
@@ -0,0 +1,231 @@
+/*
+ * Copyright 2017, Yahoo, Inc.
+ * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
+ */
+
+package com.yahoo.sketches.matrix;
+
+import static com.yahoo.memory.UnsafeUtil.unsafe;
+
+import com.yahoo.memory.Memory;
+import com.yahoo.sketches.MatrixFamily;
+import com.yahoo.sketches.SketchesArgumentException;
+
+/**
+ * This class defines the preamble items structure and provides basic utilities for some of the
+ * key fields for a Matrix
+ *
+ * <p>
+ * The low significance bytes of this <tt>long</tt> items structure are on the right. Multi-byte
+ * integers (<tt>int</tt> and <tt>long</tt>) are stored in native byte order. All <tt>byte</tt>
+ * values are treated as unsigned.</p>
+ *
+ * <p>An empty or non-compact Matrix requires 16 bytes. A compact under-full matrix requires
+ * 24 bytes of preamble.</p>
+ *
+ * <pre>
+ * Long || Start Byte Adr:
+ * Adr:
+ * || 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 |
+ * 0 ||-------------Reserved--------------| Flags | FamID | SerVer | Preamble_Longs |
+ *
+ * || 15 | 14 | 13 | 12 | 11 | 10 | 9 | 8 |
+ * 1 ||-----------Num. Columns------------|-------------Num. Rows-------------------------|
+ *
+ * || 23 | 22 | 21 | 20 | 19 | 18 | 17 | 16 |
+ * 2 ||---------Num. Columns Used---------|----------Num. Rows Used-----------------------|
+ * </pre>
+ *
+ * @author Jon Malkin
+ */
+public final class MatrixPreambleUtil {
+
+ /**
+ * The java line separator character as a String.
+ */
+ public static final String LS = System.getProperty("line.separator");
+
+ private MatrixPreambleUtil() {}
+
+ // ###### DO NOT MESS WITH THIS FROM HERE ...
+ // Preamble byte Addresses
+ private static final int PREAMBLE_LONGS_BYTE = 0;
+ private static final int SER_VER_BYTE = 1;
+ private static final int FAMILY_BYTE = 2;
+ private static final int FLAGS_BYTE = 3;
+ private static final int NUM_ROWS_INT = 8;
+ private static final int NUM_COLUMNS_INT = 12;
+ private static final int ROWS_USED_INT = 16;
+ private static final int COLS_USED_INT = 20;
+
+ // flag bit masks
+ //static final int EMPTY_FLAG_MASK = 4;
+ static final int COMPACT_FLAG_MASK = 8;
+
+ // Other constants
+ static final int SER_VER = 1;
+
+
+
+ /**
+ * Returns a human readable string summary of the preamble state of the given byte array.
+ * Used primarily in testing.
+ *
+ * @param byteArr the given byte array.
+ * @return the summary preamble string.
+ */
+ public static String preambleToString(final byte[] byteArr) {
+ final Memory mem = Memory.wrap(byteArr);
+ return preambleToString(mem);
+ }
+
+ /**
+ * Returns a human readable string summary of the preamble state of the given Memory.
+ * Note: other than making sure that the given Memory size is large
+ * enough for just the preamble, this does not do much value checking of the contents of the
+ * preamble as this is primarily a tool for debugging the preamble visually.
+ *
+ * @param mem the given Memory.
+ * @return the summary preamble string.
+ */
+ private static String preambleToString(final Memory mem) {
+
+ final int preLongs = getAndCheckPreLongs(mem); // make sure we can get the assumed preamble
+ final MatrixFamily family = MatrixFamily.idToFamily(mem.getByte(FAMILY_BYTE));
+ if (family != MatrixFamily.MATRIX) {
+ throw new SketchesArgumentException("Invalid family in memory region. Expected Matrix; "
+ + "Found: " + family.getFamilyName());
+ }
+
+ final int serVer = extractSerVer(mem);
+ if (serVer != SER_VER) {
+ throw new SketchesArgumentException("Invalid serialization version in memory region. "
+ + "Found: " + serVer);
+ }
+
+ final int flags = extractFlags(mem);
+ final String flagsStr = Integer.toBinaryString(flags) + ", " + flags;
+ //final boolean isEmpty = (flags & EMPTY_FLAG_MASK) > 0;
+ final boolean isCompact = (flags & COMPACT_FLAG_MASK) > 0;
+
+ final int numRows = extractNumRows(mem);
+ final int numCols = extractNumColumns(mem);
+
+ int numRowsUsed = numRows;
+ int numColsUsed = numCols;
+ if (isCompact) {
+ numRowsUsed = extractNumRowsUsed(mem);
+ numColsUsed = extractNumColumnsUsed(mem);
+ }
+
+ final StringBuilder sb = new StringBuilder();
+ sb.append(LS)
+ .append("### START ")
+ .append(family.getFamilyName().toUpperCase())
+ .append(" PREAMBLE SUMMARY").append(LS)
+ .append("Byte 0: Preamble Longs : ").append(preLongs).append(LS)
+ .append("Byte 1: Serialization Version: ").append(serVer).append(LS)
+ .append("Byte 2: Family : ").append(family.toString()).append(LS)
+ .append("Byte 3: Flags Field : ").append(flagsStr).append(LS)
+ //.append(" EMPTY : ").append(isEmpty).append(LS)
+ .append(" COMPACT : ").append(isCompact).append(LS)
+ .append("Bytes 8-11: Num Rows : ").append(numRows).append(LS)
+ .append("Bytes 12-15: Num Columns : ").append(numCols).append(LS);
+
+ if (isCompact) {
+ sb.append("Bytes 16-23: Num Rows Used : ").append(numRowsUsed).append(LS);
+ sb.append("Bytes 24-31: Num Columns Used : ").append(numColsUsed).append(LS);
+ }
+
+ return sb.toString();
+ }
+
+ // Extraction methods
+
+ static int extractPreLongs(final Memory mem) {
+ return mem.getInt(PREAMBLE_LONGS_BYTE) & 0xFF;
+ }
+
+ static int extractSerVer(final Memory mem) {
+ return mem.getInt(SER_VER_BYTE) & 0xFF;
+ }
+
+ static int extractFamilyID(final Memory mem) {
+ return mem.getByte(FAMILY_BYTE) & 0xFF;
+ }
+
+ static int extractFlags(final Memory mem) {
+ return mem.getByte(FLAGS_BYTE) & 0xFF;
+ }
+
+ static int extractNumRows(final Memory mem) {
+ return mem.getInt(NUM_ROWS_INT);
+ }
+
+ static int extractNumColumns(final Memory mem) {
+ return mem.getInt(NUM_COLUMNS_INT);
+ }
+
+ static int extractNumRowsUsed(final Memory mem) {
+ return mem.getInt(ROWS_USED_INT);
+ }
+
+ static int extractNumColumnsUsed(final Memory mem) {
+ return mem.getInt(COLS_USED_INT);
+ }
+
+ // Insertion methods
+
+ static void insertPreLongs(final Object memObj, final long memAddr, final int preLongs) {
+ unsafe.putByte(memObj, memAddr + PREAMBLE_LONGS_BYTE, (byte) preLongs);
+ }
+
+ static void insertSerVer(final Object memObj, final long memAddr, final int serVer) {
+ unsafe.putByte(memObj, memAddr + SER_VER_BYTE, (byte) serVer);
+ }
+
+ static void insertFamilyID(final Object memObj, final long memAddr, final int matrixFamId) {
+ unsafe.putByte(memObj, memAddr + FAMILY_BYTE, (byte) matrixFamId);
+ }
+
+ static void insertFlags(final Object memObj, final long memAddr, final int flags) {
+ unsafe.putByte(memObj, memAddr + FLAGS_BYTE, (byte) flags);
+ }
+
+ static void insertNumRows(final Object memObj, final long memAddr, final int numRows) {
+ unsafe.putInt(memObj, memAddr + NUM_ROWS_INT, numRows);
+ }
+
+ static void insertNumColumns(final Object memObj, final long memAddr, final int numColumns) {
+ unsafe.putInt(memObj, memAddr + NUM_COLUMNS_INT, numColumns);
+ }
+
+ static void insertNumRowsUsed(final Object memObj, final long memAddr, final int rowsUsed) {
+ unsafe.putInt(memObj, memAddr + ROWS_USED_INT, rowsUsed);
+ }
+
+ static void insertNumColumnsUsed(final Object memObj, final long memAddr, final int columnsUsed) {
+ unsafe.putInt(memObj, memAddr + COLS_USED_INT, columnsUsed);
+ }
+
+
+ /**
+ * Checks Memory for capacity to hold the preamble and returns the extracted preLongs.
+ * @param mem the given Memory
+ * @return the extracted prelongs value.
+ */
+ static int getAndCheckPreLongs(final Memory mem) {
+ final long cap = mem.getCapacity();
+ if (cap < 8) { throwNotBigEnough(cap, 8); }
+ final int preLongs = mem.getByte(0) & 0x3F;
+ final int required = Math.max(preLongs << 3, 8);
+ if (cap < required) { throwNotBigEnough(cap, required); }
+ return preLongs;
+ }
+
+ private static void throwNotBigEnough(final long cap, final int required) {
+ throw new SketchesArgumentException(
+ "Possible Corruption: Size of byte array or Memory not large enough: Size: " + cap
+ + ", Required: " + required);
+ }
+}
diff --git a/src/main/java/com/yahoo/sketches/matrix/package-info.java b/src/main/java/com/yahoo/sketches/matrix/package-info.java
new file mode 100644
index 0000000..ec7459c
--- /dev/null
+++ b/src/main/java/com/yahoo/sketches/matrix/package-info.java
@@ -0,0 +1,14 @@
+/*
+ * Copyright 2017, Yahoo! Inc.
+ * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root
+ * for terms.
+ */
+
+/**
+ * <p>This package contains a Matrix class that wraps one of several underlying matrix
+ * implementations. It can be used to provide a stable public API independent of the
+ * specific linear algebra package used for computation.</p>
+ *
+ * <p>These Matrix objects can be serialized and deserialized to/from a compact form.</p>
+ */
+package com.yahoo.sketches.matrix;
\ No newline at end of file
diff --git a/src/test/java/com/yahoo/sketches/decomposition/FrequentDirectionsTest.java b/src/test/java/com/yahoo/sketches/decomposition/FrequentDirectionsTest.java
new file mode 100644
index 0000000..dc515f2
--- /dev/null
+++ b/src/test/java/com/yahoo/sketches/decomposition/FrequentDirectionsTest.java
@@ -0,0 +1,337 @@
+package com.yahoo.sketches.decomposition;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+import java.util.Arrays;
+
+import com.yahoo.memory.Memory;
+import com.yahoo.memory.WritableMemory;
+import com.yahoo.sketches.MatrixFamily;
+import com.yahoo.sketches.SketchesArgumentException;
+import com.yahoo.sketches.matrix.Matrix;
+import org.testng.annotations.Test;
+
+public class FrequentDirectionsTest {
+ @Test
+ public void instantiateFD() {
+ final int k = 32;
+ final int d = 256;
+ final FrequentDirections fd = FrequentDirections.newInstance(k, d);
+
+ assertNotNull(fd);
+ assertTrue(fd.isEmpty());
+ assertEquals(fd.getK(), k);
+ assertEquals(fd.getD(), d);
+ assertEquals(fd.getN(), 0);
+ assertNull(fd.getResult());
+
+ // error conditions
+ // d = 0
+ try {
+ FrequentDirections.newInstance(k, 0);
+ fail();
+ } catch (final IllegalArgumentException e) {
+ // expected
+ }
+
+ // k = -1
+ try {
+ FrequentDirections.newInstance(-1, d);
+ fail();
+ } catch (final IllegalArgumentException e) {
+ // expected
+ }
+
+ // d < 2k (not handled in reduceRank()
+ try {
+ FrequentDirections.newInstance(d, d);
+ fail();
+ } catch (final IllegalArgumentException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void checkUpdate() {
+ final int k = 4;
+ final int d = 16; // should be > 2k
+ final FrequentDirections fd = FrequentDirections.newInstance(k, d);
+
+ // creates matrix with increasing values along diagonal
+ final double[] input = new double[d];
+ for (int i = 0; i < 2 * k; ++i) {
+ if (i > 0) {
+ input[i - 1] = 0.0;
+ }
+ //input[i] = (2 * k) - (i * 1.0);
+ input[i] = i * 1.0;
+ fd.update(input);
+ }
+ fd.update((double[]) null); // should be a no-op and not impact next lines
+ assertEquals(fd.getNumRows(), 2 * k);
+ assertEquals(fd.getN(), 2 * k);
+
+ input[(2 * k) - 1] = 0.0;
+ input[2 * k] = 2.0 * k;
+ fd.update(input); // trigger reduceRank(), then add 1 more row
+ assertEquals(fd.getNumRows(), k + 1);
+
+ fd.reset();
+ assertTrue(fd.isEmpty());
+
+ println(fd.toString());
+ println(fd.toString(true));
+ }
+
+ @Test
+ public void updateWithTooFewDimensions() {
+ final int k = 4;
+ final int d = 16; // should be > 2k
+ final FrequentDirections fd = FrequentDirections.newInstance(k, d);
+
+ final double[] input = new double[d - 3];
+ try {
+ fd.update(input);
+ fail();
+ } catch (final IllegalArgumentException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void mergeSketches() {
+ final int k = 5;
+ final int d = 12; // should be > 2k
+ final int initialRows = 7;
+ final FrequentDirections fd1 = FrequentDirections.newInstance(k, d);
+ final FrequentDirections fd2 = FrequentDirections.newInstance(k, d);
+
+ // two diagonal matrices
+ final double[] input = new double[d];
+ for (int i = 0; i < initialRows; ++i) {
+ if (i > 0) {
+ input[i - 1] = 0.0;
+ }
+ //input[i] = (2 * k) - (i * 1.0);
+ input[i] = i * 1.0;
+ fd1.update(input);
+
+ input[i] = (i * 1.0) - (2 * k);
+ fd2.update(input);
+ }
+
+ // the next two lines are no-ops
+ fd1.update((FrequentDirections) null);
+ fd1.update(FrequentDirections.newInstance(k, d));
+ assertEquals(fd1.getNumRows(), initialRows);
+ assertEquals(fd1.getN(), initialRows);
+
+ assertEquals(fd2.getNumRows(), initialRows);
+ assertEquals(fd2.getN(), initialRows);
+
+ fd1.update(fd2);
+ final int expectedRows = ((2 * initialRows) % k) + k; // assumes 2 * initialRows > k
+ assertEquals(fd1.getNumRows(), expectedRows);
+ assertEquals(fd1.getN(), 2 * initialRows);
+
+ Matrix result = fd1.getResult(false, false);
+ assertNotNull(result);
+ assertEquals(result.getNumRows(), expectedRows);
+
+ result = fd1.getResult(true, true);
+ assertNotNull(result);
+ assertEquals(result.getNumRows(), k);
+
+ println(fd1.toString(true, true, true));
+ }
+
+ @Test
+ public void checkCompensativeResult() {
+ final int k = 4;
+ final int d = 10; // should be > 2k
+ final FrequentDirections fd = FrequentDirections.newInstance(k, d);
+
+ // diagonal matrix for easy checking
+ final double[] input = new double[d];
+ for (int i = 0; i < k + 1; ++i) {
+ if (i > 0) {
+ input[i - 1] = 0.0;
+ }
+ input[i] = (i + 1) * 1.0;
+ fd.update(input);
+ }
+
+ Matrix m = fd.getResult(false, false);
+ for (int i = 0; i < k + 1; ++i) {
+ assertEquals(m.getElement(i,i), 1.0 * (i + 1), 1e-6);
+ }
+
+ final Matrix p = fd.getProjectionMatrix();
+ double[] sv = fd.getSingularValues(false);
+
+ // without compensation, and check projection at the same time
+ m = fd.getResult(true, false);
+ for (int i = k; i > 1; --i) {
+ final double val = Math.abs(m.getElement(k - i, i));
+ final double expected = Math.sqrt(((i + 1) * (i + 1)) - fd.getSvAdjustment());
+ assertEquals(val, expected, 1e-6);
+ assertEquals(sv[k - i], expected, 1e-10);
+ assertEquals(Math.abs(p.getElement(k - i, i)), 1.0, 1e-6);
+ }
+ assertEquals(m.getElement(k, 1), 0.0);
+ assertEquals(p.getElement(k, 1), 0.0);
+
+ // with compensation
+ m = fd.getResult(true, true);
+ sv = fd.getSingularValues();
+ for (int i = k; i > 1; --i) {
+ final double val = Math.abs(m.getElement(k - i, i));
+ assertEquals(val, i + 1.0, 1e-6);
+ assertEquals(sv[k - i], i + 1.0, 1e-10);
+ }
+ assertEquals(m.getElement(k, 1), 0.0);
+ }
+
+ @Test
+ public void mergeIncompatibleSketches() {
+ final int k = 5;
+ final int d = 12; // should be > 2k
+ final FrequentDirections fd1 = FrequentDirections.newInstance(k, d);
+
+ final double[] input = new double[d];
+ input[0] = 1.0;
+ fd1.update(input);
+
+ // merge in smaller k
+ FrequentDirections fd2 = FrequentDirections.newInstance(k - 1, d);
+ fd2.update(input);
+ try {
+ fd1.update(fd2);
+ fail();
+ } catch (final SketchesArgumentException e) {
+ // expected
+ }
+
+ // mismatch in d
+ fd2 = FrequentDirections.newInstance(k, d - 1);
+ fd2.update(new double[d - 1]);
+ try {
+ fd1.update(fd2);
+ fail();
+ } catch (final SketchesArgumentException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void checkSerialization() {
+ final int k = 7;
+ final int d = 20;
+ final FrequentDirections fd = FrequentDirections.newInstance(k, d);
+
+ byte[] sketchBytes = fd.toByteArray(false);
+ assertEquals(sketchBytes.length,
+ MatrixFamily.FREQUENTDIRECTIONS.getMinPreLongs() * Long.BYTES);
+ Memory mem = Memory.wrap(sketchBytes);
+ FrequentDirections rebuilt = FrequentDirections.heapify(mem);
+ assertTrue(rebuilt.isEmpty());
+ assertEquals(rebuilt.getD(), fd.getD());
+ assertEquals(rebuilt.getK(), fd.getK());
+
+ // creates matrix with increasing values along diagonal
+ // k rows, so shouldn't compress yet
+ final double[] input = new double[d];
+ for (int i = 0; i < k; ++i) {
+ if (i > 0) {
+ input[i - 1] = 0.0;
+ }
+ //input[i] = (2 * k) - (i * 1.0);
+ input[i] = i * 1.0;
+ fd.update(input);
+ }
+ sketchBytes = fd.toByteArray();
+ mem = Memory.wrap(sketchBytes);
+ rebuilt = FrequentDirections.heapify(mem);
+ assertEquals(rebuilt.getN(), fd.getN());
+ assertEquals(rebuilt.getD(), fd.getD());
+ assertEquals(rebuilt.getK(), fd.getK());
+
+ // add another k rows and serialize, compressing this time
+ for (int i = k; i < 2 * k - 1; ++i) {
+ input[i] = i * 1.0;
+ fd.update(input);
+ }
+ assertEquals(fd.getNumRows(), 2 * k - 1);
+ sketchBytes = fd.toByteArray(true);
+ assertEquals(fd.getNumRows(), k);
+ mem = Memory.wrap(sketchBytes);
+ rebuilt = FrequentDirections.heapify(mem);
+ assertEquals(rebuilt.getN(), fd.getN());
+ assertEquals(rebuilt.getNumRows(), k);
+
+ println(PreambleUtil.preambleToString(mem));
+ }
+
+ @Test
+ public void checkCorruptedHeapify() {
+ final int k = 50;
+ final int d = 250;
+ final FrequentDirections fd = FrequentDirections.newInstance(k, d);
+ byte[] sketchBytes = fd.toByteArray();
+ WritableMemory mem = WritableMemory.wrap(sketchBytes);
+
+ FrequentDirections rebuilt = FrequentDirections.heapify(mem);
+ assertTrue(rebuilt.isEmpty());
+ println(PreambleUtil.preambleToString(mem));
+
+ // corrupt the serialization version
+ mem.putByte(PreambleUtil.SER_VER_BYTE, (byte) 0);
+ try {
+ FrequentDirections.heapify(mem);
+ } catch (final SketchesArgumentException e) {
+ // expected
+ }
+
+ // corrupt the family ID, after grabbing fresh bytes
+ sketchBytes = fd.toByteArray();
+ mem = WritableMemory.wrap(sketchBytes);
+ mem.putByte(PreambleUtil.FAMILY_BYTE, (byte) 0);
+ try {
+ FrequentDirections.heapify(mem);
+ } catch (final SketchesArgumentException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void checkInsufficientMemory() {
+ // no capacity
+ byte[] bytes = new byte[0];
+ Memory mem = Memory.wrap(bytes);
+ try {
+ FrequentDirections.heapify(mem);
+ } catch (final SketchesArgumentException e) {
+ // expected
+ }
+
+ // capacity smaller than prelongs size
+ final FrequentDirections fd = FrequentDirections.newInstance(10, 50);
+ bytes = fd.toByteArray();
+ bytes = Arrays.copyOf(bytes, bytes.length - 1);
+ mem = Memory.wrap(bytes);
+ try {
+ FrequentDirections.heapify(mem);
+ } catch (final SketchesArgumentException e) {
+ // expected
+ }
+ }
+
+
+ private void println(final String msg) {
+ //System.out.println(msg);
+ }
+}
diff --git a/src/test/java/com/yahoo/sketches/matrix/MatrixImplOjAlgoTest.java b/src/test/java/com/yahoo/sketches/matrix/MatrixImplOjAlgoTest.java
new file mode 100644
index 0000000..442d674
--- /dev/null
+++ b/src/test/java/com/yahoo/sketches/matrix/MatrixImplOjAlgoTest.java
@@ -0,0 +1,206 @@
+package com.yahoo.sketches.matrix;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.fail;
+
+import com.yahoo.memory.Memory;
+import com.yahoo.memory.WritableMemory;
+import com.yahoo.sketches.SketchesArgumentException;
+import org.ojalgo.matrix.store.PrimitiveDenseStore;
+import org.testng.annotations.Test;
+
+public class MatrixImplOjAlgoTest {
+ @Test
+ public void checkInstantiation() {
+ final int nRows = 10;
+ final int nCols = 15;
+ final Matrix m = MatrixImplOjAlgo.newInstance(nRows, nCols);
+ assertEquals(m.getNumRows(), nRows);
+ assertEquals(m.getNumColumns(), nCols);
+
+ final PrimitiveDenseStore pds = (PrimitiveDenseStore) m.getRawObject();
+ assertEquals(pds.countRows(), nRows);
+ assertEquals(pds.countColumns(), nCols);
+
+ final Matrix wrapped = Matrix.wrap(pds);
+ MatrixTest.checkMatrixEquality(wrapped, m);
+ assertEquals(wrapped.getRawObject(), pds);
+ }
+
+ @Test
+ public void updateAndQueryValues() {
+ final int nRows = 5;
+ final int nCols = 5;
+ final Matrix m = generateIncreasingEye(nRows, nCols); // tests setElement() in method
+
+ for (int i = 0; i < nRows; ++i) {
+ for (int j = 0; j < nCols; ++j) {
+ double val = m.getElement(i, j);
+ if (i == j) {
+ assertEquals(val, i + 1.0);
+ } else {
+ assertEquals(val, 0.0);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void checkStandardSerialization() {
+ final int nRows = 3;
+ final int nCols = 7;
+ final Matrix m = generateIncreasingEye(nRows, nCols);
+
+ final byte[] mtxBytes = m.toByteArray();
+ assertEquals(mtxBytes.length, m.getSizeBytes());
+
+ final Memory mem = Memory.wrap(mtxBytes);
+ final Matrix tgt = MatrixImplOjAlgo.heapifyInstance(mem);
+ MatrixTest.checkMatrixEquality(tgt, m);
+ }
+
+ @Test
+ public void checkCompactSerialization() {
+ final int nRows = 4;
+ final int nCols = 7;
+ final Matrix m = generateIncreasingEye(nRows, nCols);
+
+ byte[] mtxBytes = m.toCompactByteArray(nRows - 1, 7);
+ assertEquals(mtxBytes.length, m.getCompactSizeBytes(nRows - 1, 7));
+
+ Memory mem = Memory.wrap(mtxBytes);
+ Matrix tgt = MatrixImplOjAlgo.heapifyInstance(mem);
+ for (int c = 0; c < nCols; ++c) {
+ for (int r = 0; r < nRows - 1; ++r) {
+ assertEquals(tgt.getElement(r, c), m.getElement(r, c)); // equal here
+ }
+ // assuming nRows - 1 so check only the last row as being 0
+ assertEquals(tgt.getElement(nRows - 1, c), 0.0);
+ }
+
+ // test without compacting
+ mtxBytes = m.toCompactByteArray(nRows, nCols);
+ assertEquals(mtxBytes.length, m.getSizeBytes());
+ mem = Memory.wrap(mtxBytes);
+ tgt = MatrixImplOjAlgo.heapifyInstance(mem);
+ MatrixTest.checkMatrixEquality(tgt, m);
+ }
+
+ @Test
+ public void matrixRowOperations() {
+ final int nRows = 7;
+ final int nCols = 5;
+ final Matrix m = generateIncreasingEye(nRows, nCols);
+
+ final int tgtCol = 2;
+ final double[] v = m.getRow(tgtCol); // diagonal matrix, so this works ok
+ for (int i = 0; i < v.length; ++i) {
+ assertEquals(v[i], (i == tgtCol ? i + 1.0 : 0.0));
+ }
+
+ assertEquals(m.getElement(6, tgtCol), 0.0);
+ m.setRow(6, v);
+ assertEquals(m.getElement(6, tgtCol), tgtCol + 1.0);
+ }
+
+ @Test
+ public void matrixColumnOperations() {
+ final int nRows = 9;
+ final int nCols = 4;
+ final Matrix m = generateIncreasingEye(nRows, nCols);
+
+ final int tgtRow = 3;
+ final double[] v = m.getColumn(tgtRow); // diagonal matrix, so this works ok
+ for (int i = 0; i < v.length; ++i) {
+ assertEquals(v[i], (i == tgtRow ? i + 1.0 : 0.0));
+ }
+
+ assertEquals(m.getElement(tgtRow, 0), 0.0);
+ m.setColumn(0, v);
+ assertEquals(m.getElement(tgtRow, 0), tgtRow + 1.0);
+ }
+
+ @Test
+ public void invalidRowColumnOperations() {
+ final int nRows = 9;
+ final int nCols = 4;
+ final Matrix m = generateIncreasingEye(nRows, nCols);
+
+ final double[] shortRow = new double[nCols - 2];
+ try {
+ m.setRow(1, shortRow);
+ fail();
+ } catch (final SketchesArgumentException e) {
+ // expected
+ }
+
+ final double[] longColumn = new double[nRows + 2];
+ try {
+ m.setColumn(1, longColumn);
+ fail();
+ } catch (final SketchesArgumentException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void invalidSerVer() {
+ final int nRows = 3;
+ final int nCols = 3;
+ final Matrix m = generateIncreasingEye(nRows, nCols);
+ final byte[] sketchBytes = m.toByteArray();
+ final WritableMemory mem = WritableMemory.wrap(sketchBytes);
+ MatrixPreambleUtil.insertSerVer(mem.getArray(), mem.getCumulativeOffset(0L), 0);
+
+ try {
+ MatrixImplOjAlgo.heapifyInstance(mem);
+ fail();
+ } catch (final SketchesArgumentException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void invalidFamily() {
+ final int nRows = 3;
+ final int nCols = 3;
+ final Matrix m = generateIncreasingEye(nRows, nCols);
+ final byte[] sketchBytes = m.toByteArray();
+ final WritableMemory mem = WritableMemory.wrap(sketchBytes);
+ MatrixPreambleUtil.insertFamilyID(mem.getArray(), mem.getCumulativeOffset(0L), 0);
+
+ try {
+ MatrixImplOjAlgo.heapifyInstance(mem);
+ fail();
+ } catch (final SketchesArgumentException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void insufficientMemoryCapacity() {
+ final byte[] bytes = new byte[6];
+ final Memory mem = Memory.wrap(bytes);
+ try {
+ MatrixImplOjAlgo.heapifyInstance(mem);
+ fail();
+ } catch (final SketchesArgumentException e) {
+ // expected
+ }
+ }
+
+ /**
+ * Creates a scaled I matrix, where the diagonal consists of increasing integers,
+ * starting with 1.0.
+ * @param nRows number of rows
+ * @param nCols number of columns
+ * @return PrimitiveDenseStore, suitable for direct use or wrapping
+ */
+ private Matrix generateIncreasingEye(final int nRows, final int nCols) {
+ final Matrix m = MatrixImplOjAlgo.newInstance(nRows, nCols);
+ for (int i = 0; i < nRows && i < nCols; ++i) {
+ m.setElement(i, i, 1.0 + i);
+ }
+ return m;
+ }
+}
diff --git a/src/test/java/com/yahoo/sketches/matrix/MatrixTest.java b/src/test/java/com/yahoo/sketches/matrix/MatrixTest.java
new file mode 100644
index 0000000..742626d
--- /dev/null
+++ b/src/test/java/com/yahoo/sketches/matrix/MatrixTest.java
@@ -0,0 +1,94 @@
+package com.yahoo.sketches.matrix;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+import com.yahoo.memory.Memory;
+import com.yahoo.sketches.MatrixFamily;
+import com.yahoo.sketches.SketchesArgumentException;
+import org.testng.annotations.Test;
+
+public class MatrixTest {
+
+ @Test
+ public void checkHeapify() {
+ final Matrix m = Matrix.builder().setType(MatrixBuilder.Algo.OJALGO).build(3, 3);
+ final byte[] bytes = m.toByteArray();
+ final Memory mem = Memory.wrap(bytes);
+
+ Matrix tgt = Matrix.heapify(mem, MatrixBuilder.Algo.OJALGO);
+ assertTrue(tgt instanceof MatrixImplOjAlgo);
+ checkMatrixEquality(m, tgt);
+
+ tgt = Matrix.heapify(mem, MatrixBuilder.Algo.NATIVE);
+ assertNull(tgt);
+ }
+
+ @Test
+ public void checkWrap() {
+ assertNull(Matrix.wrap(null));
+
+ final Matrix src = Matrix.builder().setType(MatrixBuilder.Algo.OJALGO).build(3, 3);
+ final Object obj = src.getRawObject();
+ final Matrix tgt = Matrix.wrap(obj);
+ assertTrue(tgt instanceof MatrixImplOjAlgo);
+ checkMatrixEquality(src, tgt);
+
+ try {
+ final Object notAMatrix = 1.0;
+ Matrix.wrap(notAMatrix);
+ fail();
+ } catch (final SketchesArgumentException e) {
+ // expected
+ }
+
+ assertNotNull(src.toString());
+ }
+
+ @Test
+ public void checkSize() {
+ final int nRow = 7;
+ final int nCol = 3;
+ final Matrix m = Matrix.builder().build(nRow, nCol);
+
+ int expectedSize = (MatrixFamily.MATRIX.getMinPreLongs() * Long.BYTES)
+ + (nRow * nCol * Double.BYTES);
+ assertEquals(m.getSizeBytes(), expectedSize);
+
+ // this should redirect call to getSizeBytes()
+ assertEquals(m.getCompactSizeBytes(nRow, nCol), expectedSize);
+
+ // degenerate cases
+ expectedSize = (MatrixFamily.MATRIX.getMinPreLongs() * Long.BYTES);
+ assertEquals(m.getCompactSizeBytes(0, nCol), expectedSize);
+ assertEquals(m.getCompactSizeBytes(nRow, 0), expectedSize);
+
+ // matrix subsets
+ expectedSize = (MatrixFamily.MATRIX.getMaxPreLongs() * Long.BYTES)
+ + (5 * 3) * Double.BYTES;
+ assertEquals(m.getCompactSizeBytes(5, 3), expectedSize);
+
+ expectedSize = (MatrixFamily.MATRIX.getMaxPreLongs() * Long.BYTES)
+ + (7 * 2) * Double.BYTES;
+ assertEquals(m.getCompactSizeBytes(7, 2), expectedSize);
+
+ expectedSize = (MatrixFamily.MATRIX.getMaxPreLongs() * Long.BYTES)
+ + (2 * 2) * Double.BYTES;
+ assertEquals(m.getCompactSizeBytes(2, 2), expectedSize);
+ }
+
+ static void checkMatrixEquality(final Matrix m1, final Matrix m2) {
+ assertEquals(m1.numRows_, m2.numRows_);
+ assertEquals(m1.numCols_, m2.numCols_);
+
+ for (int i = 0; i < m1.numRows_; ++i) {
+ for (int j = 0; j < m1.numCols_; ++j) {
+ assertEquals(m1.getElement(i, j), m2.getElement(i, j),
+ "Mismatch at (" + i + ", " + j + ")");
+ }
+ }
+ }
+}
diff --git a/tools/CloverConfig.txt b/tools/CloverConfig.txt
new file mode 100644
index 0000000..9e0ec43
--- /dev/null
+++ b/tools/CloverConfig.txt
@@ -0,0 +1,33 @@
+Clover Config for Eclipse:
+
+At Project Level Properties:
+Clover:
+ Instrumentation:
+ Initstring: Default value
+ Output Folder: ...project output dir(s)
+ Flush Policy: At JVM shutdown ...
+ Misc: Fully qualify ... , Instrument and compile at statement level
+ Contexts:
+ Check: assert statements
+ Add Custom Coverage Context Filter:
+ private-constructor: also see link below
+ Method
+ (.* )?private +[a-zA-Z0-9_$]+ *\( *\).*
+ Source Files
+ Only look ...
+ [check] src/main/java[includes=**/*.java][excludes=]
+ [check] src/test/java[includes=**/*.java][excludes=]
+ Test Classes
+ Assume all source in the specified folders are tests or test utility classes
+ [check] src/test/java
+
+At Clover "down-triangle" menu:
+ Columns:
+ Element
+ % TOTAL Coverage
+ Uncovered Elements: Custom: %UncoveredElements * TotalElements / 100
+ Total Elements
+
+
+
+http://alexruizlog.blogspot.com/2009/04/how-to-make-clover-ignore-private_21.html
\ No newline at end of file
diff --git a/tools/FindBugsExcludeFilter.xml b/tools/FindBugsExcludeFilter.xml
new file mode 100644
index 0000000..16e5fa3
--- /dev/null
+++ b/tools/FindBugsExcludeFilter.xml
@@ -0,0 +1,90 @@
+<FindBugsFilter> <!-- sketches-core -->
+
+ <!-- Too many false positives to be useful. I could not make it happy :( -->
+ <Match>
+ <Bug pattern="SF_SWITCH_NO_DEFAULT" />
+ </Match>
+
+ <!-- False positive. FindBugs complains if DQS_RESIZE_THRESHOLD == REBUILD_THRESHOLD,
+ but this allows us to tune these constants for different sketches. -->
+ <Match>
+ <Bug pattern="DB_DUPLICATE_BRANCHES" />
+ <Class name="com.yahoo.sketches.theta.DirectQuickSelectSketch" />
+ <Method name="setHashTableThreshold" />
+ </Match>
+
+ <!-- False positive. In this case we want to ignore the exceptions -->
+ <Match>
+ <Bug pattern="DE_MIGHT_IGNORE" />
+ <Class name="com.yahoo.sketches.theta.PairwiseCornerCasesTest" />
+ <Method name="checkNotOrdered" />
+ </Match>
+
+ <Match> <!-- Exclude for test classes; too many False Positives. -->
+ <Bug pattern="NP_NULL_PARAM_DEREF_NONVIRTUAL" />
+ <Class name="~.*\.*Test" />
+ </Match>
+
+ <Match> <!-- Exclude for test classes; too many False Positives. -->
+ <Bug pattern="DLS_DEAD_LOCAL_STORE" />
+ <Class name="~.*\.*Test" />
+ </Match>
+
+</FindBugsFilter>
+
+
+
+<!-- Examples: -->
+
+<!-- Exclude java.* classes -->
+ <!--
+ <Match>
+ <Package name="java\.*" />
+ </Match>
+-->
+
+ <!-- Exclude test classes -->
+<!--
+ <Match>
+ <Class name="~.*\.*Test" />
+ </Match>
+-->
+
+<!--
+ <Match>
+ <Class name="com.foobar.ClassNotToBeAnalyzed" />
+ </Match>
+-->
+<!--
+ <Match>
+ <Class name="com.foobar.ClassWithSomeBugsMatched" />
+ <Bug code="DE,UrF,SIC" />
+ </Match>
+-->
+ <!-- Match all XYZ violations. -->
+<!--
+ <Match>
+ <Bug code="XYZ" />
+ </Match>
+-->
+ <!-- Match all doublecheck violations in these methods of "AnotherClass". -->
+<!--
+ <Match>
+ <Class name="com.foobar.AnotherClass" />
+ <Or>
+ <Method name="nonOverloadedMethod" />
+ <Method name="frob" params="int,java.lang.String" returns="void" />
+ <Method name="blat" params="" returns="boolean" />
+ </Or>
+ <Bug code="DC" />
+ </Match>
+-->
+ <!-- A method with a dead local store false positive (medium priority). -->
+<!--
+ <Match>
+ <Class name="com.foobar.MyClass" />
+ <Method name="someMethod" />
+ <Bug pattern="DLS_DEAD_LOCAL_STORE" />
+ <Priority value="2" />
+ </Match>
+-->
diff --git a/tools/SketchesCheckstyle.xml b/tools/SketchesCheckstyle.xml
new file mode 100644
index 0000000..aaebec2
--- /dev/null
+++ b/tools/SketchesCheckstyle.xml
@@ -0,0 +1,373 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd"> <!-- does not work with https -->
+
+<!--
+ SketchesCheckstyle.xml for sketches-core
+
+ Checkstyle is very configurable. Be sure to read the documentation at
+ http://checkstyle.sourceforge.net (or in your downloaded distribution). Note: Does not work with https.
+
+ To completely disable a check, just comment it out or delete it from the file.
+
+ Authors: Max Vetrenko, Ruslan Diachenko, Roman Ivanov.
+-->
+
+<module name = "Checker">
+ <property name="charset" value="UTF-8"/>
+ <property name="severity" value="warning"/>
+ <property name="fileExtensions" value="java, properties, xml"/>
+
+ <module name="FileTabCharacter">
+ <property name="eachLine" value="true"/>
+ </module>
+
+ <module name="JavadocPackage"/>
+
+ <module name="NewlineAtEndOfFile">
+ <property name="lineSeparator" value="lf"/>
+ </module>
+
+ <module name="TreeWalker">
+
+ <!-- Enable suppression using comments: //CHECKSTYLE.OFF "RULE" and //CHECKSTYLE.ON "RULE"
+ You must specify the specific rule, as in: //CHECKSTYLE.OFF: LineLength -->
+ <module name="SuppressionCommentFilter">
+ <property name="offCommentFormat" value="CHECKSTYLE.OFF\: ([\w\|]+)"/>
+ <property name="onCommentFormat" value="CHECKSTYLE.ON\: ([\w\|]+)"/>
+ <property name="checkFormat" value="$1"/>
+ </module>
+
+ <!-- Annotations -->
+ <module name="AnnotationLocation">
+ <property name="tokens" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF"/>
+ </module>
+
+ <module name="AnnotationLocation">
+ <property name="tokens" value="VARIABLE_DEF"/>
+ <property name="allowSamelineMultipleAnnotations" value="true"/>
+ </module>
+
+ <!-- Block Checks -->
+ <module name="EmptyBlock">
+ <property name="severity" value="warning"/>
+ <property name="option" value="TEXT"/>
+ <property name="tokens" value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="EmptyCatchBlock">
+ <property name="severity" value="ignore"/>
+ <property name="exceptionVariableName" value="expected"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="LeftCurly">
+ <!-- doesn't allow for if (n == 0) { return 0.0; }, which is readable and not corruptable -->
+ <property name="severity" value="ignore"/>
+ <property name="maxLineLength" value="100"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="NeedBraces">
+ <property name="allowSingleLineStatement" value="false"/> <!-- default = false -->
+ <property name="allowEmptyLoopBody" value="false"/> <!-- default = false -->
+ </module>
+
+ <module name="RightCurly">
+ <!-- requires right curly start a new line; alone: on a line alone -->
+ <property name="severity" value="ignore"/>
+ <property name="option" value="alone"/>
+ <property name="tokens" value="CLASS_DEF, METHOD_DEF, CTOR_DEF, LITERAL_FOR, LITERAL_WHILE, LITERAL_DO, STATIC_INIT, INSTANCE_INIT"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <!-- Coding -->
+ <module name="FallThrough">
+ <property name="severity" value="warning"/>
+ <property name="reliefPattern" value="fallthru|falls? ?through|FALL[- ]?THROUGH"/>
+ <!-- <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/> -->
+ </module>
+
+ <module name="FinalLocalVariable">
+ <property name="severity" value="warning"/>
+ <property name="tokens" value="VARIABLE_DEF,PARAMETER_DEF"/>
+ </module>
+
+ <module name="IllegalTokenText">
+ <property name="severity" value="warning"/>
+ <property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
+ <property name="format" value="\\u00(08|09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
+ <property name="message" value="Avoid using corresponding octal or Unicode escape."/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="MissingSwitchDefault">
+ <!-- Too many false positives -->
+ <property name="severity" value="ignore"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="MultipleVariableDeclarations">
+ <property name="severity" value="ignore"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="NoFinalizer">
+ <property name="severity" value="ignore"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="OneStatementPerLine">
+ <property name="severity" value="ignore"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="OverloadMethodsDeclarationOrder">
+ <property name="severity" value="ignore"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="VariableDeclarationUsageDistance">
+ <property name="severity" value="ignore"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <!-- Class Design -->
+ <module name="OneTopLevelClass"/>
+
+ <module name="FinalClass"/>
+
+
+ <!-- Imports -->
+ <module name="AvoidStarImport"/>
+
+ <module name="CustomImportOrder">
+ <property name="specialImportsRegExp" value="org"/>
+ <property name="sortImportsInGroupAlphabetically" value="true"/>
+ <property name="separateLineBetweenGroups" value="true"/>
+ <!-- Google: "STATIC###SPECIAL_IMPORTS###THIRD_PARTY_PACKAGE###STANDARD_JAVA_PACKAGE" -->
+ <property name="customImportOrderRules" value="STATIC###STANDARD_JAVA_PACKAGE###SPECIAL_IMPORTS###THIRD_PARTY_PACKAGE"/>
+ </module>
+
+ <module name="RedundantImport"/>
+
+ <module name="UnusedImports"/>
+
+ <!-- Javadoc Comments -->
+ <!-- JavadocPackage under Checker -->
+ <module name="AtclauseOrder">
+ <property name="tagOrder" value="@param, @return, @throws, @deprecated"/>
+ <property name="target" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF, VARIABLE_DEF"/>
+ </module>
+
+ <module name="JavadocMethod">
+ <property name="scope" value="public"/>
+ <property name="allowMissingParamTags" value="false"/>
+ <property name="allowMissingThrowsTags" value="true"/>
+ <property name="allowMissingReturnTag" value="false"/>
+ <property name="minLineCount" value="2"/>
+ <property name="allowedAnnotations" value="Override, Test"/>
+ <property name="allowThrowsTagsForSubclasses" value="true"/>
+ </module>
+
+ <module name="JavadocParagraph"/>
+
+ <module name="JavadocTagContinuationIndentation">
+ <property name="severity" value="ignore"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="NonEmptyAtclauseDescription"/>
+
+ <module name="SingleLineJavadoc">
+ <property name="ignoreInlineTags" value="false"/>
+ </module>
+
+ <module name="SummaryJavadocCheck">
+ <property name="severity" value="ignore"/>
+ <property name="forbiddenSummaryFragments" value="^@return the *|^This method returns |^A [{]@code [a-zA-Z0-9]+[}]( is a )"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <!-- Miscellaneous -->
+ <module name="ArrayTypeStyle">
+ <property name="severity" value="ignore"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="AvoidEscapedUnicodeCharacters">
+ <property name="severity" value="warning"/>
+ <property name="allowEscapesForControlCharacters" value="true"/>
+ <property name="allowByTailComment" value="true"/>
+ <property name="allowNonPrintableEscapes" value="true"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="CommentsIndentation">
+ <property name="severity" value="warning"/>
+ <!-- <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/> -->
+ </module>
+
+ <module name="FileContentsHolder"/> <!-- Used with SuppressionCommentFilter -->
+
+ <module name="Indentation">
+ <property name="severity" value="ignore"/>
+ <property name="basicOffset" value="2"/>
+ <property name="braceAdjustment" value="0"/>
+ <property name="caseIndent" value="2"/>
+ <property name="throwsIndent" value="4"/>
+ <property name="lineWrappingIndentation" value="4"/>
+ <property name="arrayInitIndent" value="2"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="OuterTypeFilename">
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="TodoComment">
+ <property name="severity" value="ignore"/>
+ <property name="format" value="(//TODO)|(//FIXME)"/>
+ </module>
+
+ <module name="UpperEll">
+ <property name="severity" value="warning"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <!-- Modifiers -->
+ <module name="ModifierOrder">
+ <property name="severity" value="ignore"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <!-- Naming Conventions -->
+ <module name="AbbreviationAsWordInName">
+ <property name="severity" value="ignore"/>
+ <property name="ignoreFinal" value="false"/>
+ <property name="allowedAbbreviationLength" value="1"/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="ClassTypeParameterName">
+ <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
+ <message key="name.invalidPattern"
+ value="Class type name ''{0}'' must match pattern ''{1}''."/>
+ </module>
+
+ <module name="LocalVariableName">
+ <property name="severity" value="ignore"/>
+ <property name="tokens" value="VARIABLE_DEF"/>
+ <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
+ <property name="allowOneCharVarInForLoop" value="true"/>
+ <message key="name.invalidPattern"
+ value="Local variable name ''{0}'' must match pattern ''{1}''."/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="MemberName">
+ <property name="severity" value="ignore"/>
+ <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
+ <message key="name.invalidPattern"
+ value="Member name ''{0}'' must match pattern ''{1}''."/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="MethodName">
+ <property name="severity" value="ignore"/>
+ <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9_]*$"/>
+ <message key="name.invalidPattern"
+ value="Method name ''{0}'' must match pattern ''{1}''."/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="MethodTypeParameterName">
+ <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
+ <message key="name.invalidPattern"
+ value="Method type name ''{0}'' must match pattern ''{1}''."/>
+ </module>
+
+ <module name="PackageName">
+ <property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
+ <message key="name.invalidPattern"
+ value="Package name ''{0}'' must match pattern ''{1}''."/>
+ </module>
+
+ <module name="ParameterName">
+ <property name="severity" value="ignore"/>
+ <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
+ <message key="name.invalidPattern"
+ value="Parameter name ''{0}'' must match pattern ''{1}''."/>
+ <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/>
+ </module>
+
+ <module name="TypeName">
+ <message key="name.invalidPattern"
+ value="Type name ''{0}'' must match pattern ''{1}''."/>
+ </module>
+
+ <!-- Regexp -->
+ <module name="Regexp">
+ <property name="severity" value="ignore"/>
+ <property name="format" value="[ \t]+$"/>
+ <property name="illegalPattern" value="true"/>
+ <property name="message" value="Trailing whitespace"/>
+ <property name="ignoreComments" value="true"/>
+ </module>
+
+ <!-- Size Violations -->
+ <module name="LineLength">
+ <property name="severity" value="warning"/>
+ <property name="max" value="110"/>
+ <property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
+ <!-- <metadata name="net.sf.eclipsecs.core.lastEnabledSeverity" value="inherit"/> -->
+ </module>
+
+ <!-- Whitespace -->
+ <module name="EmptyLineSeparator">
+ <property name="allowNoEmptyLineBetweenFields" value="true"/>
+ </module>
+
+ <module name="GenericWhitespace">
+ <message key="ws.followed"
+ value="GenericWhitespace ''{0}'' is followed by whitespace."/>
+ <message key="ws.preceded"
+ value="GenericWhitespace ''{0}'' is preceded with whitespace."/>
+ <message key="ws.illegalFollow"
+ value="GenericWhitespace ''{0}'' should followed by whitespace."/>
+ <message key="ws.notPreceded"
+ value="GenericWhitespace ''{0}'' is not preceded with whitespace."/>
+ </module>
+
+ <module name="NoLineWrap"/> <!-- Only for import and package statements -->
+
+ <module name="MethodParamPad"/>
+
+ <module name="OperatorWrap">
+ <property name="option" value="NL"/>
+ <property name="tokens" value="BAND, BOR, BSR, BXOR, DIV, EQUAL, GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD, NOT_EQUAL, PLUS, QUESTION, SL, SR, STAR "/>
+ </module>
+
+ <module name="SeparatorWrap">
+ <property name="tokens" value="DOT"/>
+ <property name="option" value="nl"/>
+ </module>
+
+ <module name="SeparatorWrap">
+ <property name="tokens" value="COMMA"/>
+ <property name="option" value="EOL"/>
+ </module>
+
+ <module name="WhitespaceAround">
+ <property name="allowEmptyConstructors" value="true"/>
+ <property name="allowEmptyMethods" value="true"/>
+ <property name="allowEmptyTypes" value="true"/>
+ <property name="allowEmptyLoops" value="true"/>
+ <message key="ws.notPreceded" value="WhitespaceAround: ''{0}'' is not preceded with whitespace."/>
+ <message key="ws.notFollowed" value="WhitespaceAround: ''{0}'' is not followed by whitespace. Empty blocks may only be represented as '{}' when not part of a multi-block statement (4.1.3)"/>
+ </module>
+
+ </module> <!-- End of TreeWalker -->
+</module>
