/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.datasketches.cpc;

import static org.apache.datasketches.Util.zeroPad;
import static org.apache.datasketches.cpc.RuntimeAsserts.rtAssert;
import static org.apache.datasketches.cpc.RuntimeAsserts.rtAssertEquals;

import java.nio.ByteOrder;

import org.apache.datasketches.Family;
import org.apache.datasketches.SketchesArgumentException;
import org.apache.datasketches.SketchesStateException;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.WritableMemory;

//@formatter:off
/**
 * All formats are illustrated as Big-Endian, LSB on the right.
 *
 * <pre>
 * Format = EMPTY_MERGED/EMPTY_HIP: NoWindow, NoSV, HIP or NoHIP = 00X.
 * The first 8 bytes are common for all Formats.
 * PI = 2, FIcol = 0
 * Long adr ||
 *          ||    7   |    6   |    5   |    4   |    3   |    2   |    1   |    0   |
 *      0   ||---SEED HASH-----|-Flags--|-FIcol--|---lgK--|-FamID--|-SerVer-|---PI---|
 *
 *
 * Format = SPARSE_HYBRID_MERGED: {NoWindow, SV, NoHIP} = 2 = 010.
 * PI = 4, FIcol = 0
 * Long adr ||
 *          ||   15   |   14   |   13   |   12   |   11   |   10   |    9   |    8   |
 *      1   ||-------------SV Length Ints--------|--------numCoupons = numSV---------|
 *
 *          ||   23   |   22   |   21   |   20   |   19   |   18   |   17   |   16   |
 *      2   ||                                   |----------Start SV bit stream------|
 *
 *
 * Format = SPARSE_HYBRID_HIP: {NoWindow, SV, HIP} = 3 = 011
 * PI = 8, FIcol = 0
 * Long adr ||
 *          ||   15   |   14   |   13   |   12   |   11   |   10   |    9   |    8   |
 *      1   ||-------------SV Length Ints--------|--------numCoupons = numSV---------|
 *
 *          ||   23   |   22   |   21   |   20   |   19   |   18   |   17   |   16   |
 *      2   ||----------------------------------KxP----------------------------------|
 *
 *          ||   31   |   30   |   29   |   28   |   27   |   26   |   25   |   24   |
 *      3   ||-------------------------------HIP Accum-------------------------------|
 *
 *          ||   39   |   38   |   37   |   36   |   35   |   34   |   33   |   32   |
 *      4   ||                                   |----------Start of SV stream-------|
 *
 *
 * Format = PINNED_SLIDING_MERGED_NOSV: {Window, No SV, NoHIP} = 4 = 100
 * PI = 4, FIcol = valid
 * Long adr ||
 *          ||   15   |   14   |   13   |   12   |   11   |   10   |    9   |    8   |
 *      1   ||----------Window Length Ints-------|------------numCoupons-------------|
 *
 *          ||   23   |   22   |   21   |   20   |   19   |   18   |   17   |   16   |
 *      2   ||                                   |--------Start of Window stream-----|
 *
 *
 * Format = PINNED_SLIDING_HIP_NOSV: {Window, No SV, HIP} = 5 = 101
 * PI = 8, FIcol = valid
 * Long adr ||
 *          ||   15   |   14   |   13   |   12   |   11   |   10   |    9   |    8   |
 *      1   ||----------Window Length Ints-------|------------numCoupons-------------|
 *
 *          ||   23   |   22   |   21   |   20   |   19   |   18   |   17   |   16   |
 *      2   ||----------------------------------KxP----------------------------------|
 *
 *          ||   31   |   30   |   29   |   28   |   27   |   26   |   25   |   24   |
 *      3   ||-------------------------------HIP Accum-------------------------------|
 *
 *          ||   39   |   38   |   37   |   36   |   35   |   34   |   33   |   32   |
 *      4   ||                                   |--------Start of Window stream-----|
 *
 *
 * Format = PINNED_SLIDING_MERGED: {Window, SV, NoHIP} = 6 = 110
 * PI = 6, FIcol = valid
 * Long adr ||
 *          ||   15   |   14   |   13   |   12   |   11   |   10   |    9   |    8   |
 *      1   ||---------------numSV---------------|------------numCoupons-------------|
 *
 *          ||   23   |   22   |   21   |   20   |   19   |   18   |   17   |   16   |
 *      2   ||----------Window Length Ints-------|-------------SV Length Ints--------|
 *
 *          ||   XX   |   XX   |   XX   |   XX   |   27   |   26   |   25   |   24   |
 *      3   ||--------Start of SV stream---------|--------Start of Window stream-----|
 *
 *
 * Format = PINNED_SLIDING_HIP: {Window, SV, HIP} = 7 = 111
 * PI = 10, FIcol = valid
 * Long adr ||
 *          ||   15   |   14   |   13   |   12   |   11   |   10   |    9   |    8   |
 *      1   ||---------------numSV---------------|------------numCoupons-------------|
 *
 *          ||   23   |   22   |   21   |   20   |   19   |   18   |   17   |   16   |
 *      2   ||----------------------------------KxP----------------------------------|
 *
 *          ||   31   |   30   |   29   |   28   |   27   |   26   |   25   |   24   |
 *      3   ||-------------------------------HIP Accum-------------------------------|
 *
 *          ||   39   |   38   |   37   |   36   |   35   |   34   |   33   |   32   |
 *      4   ||----------Window Length Ints-------|-------------SV Length Ints--------|
 *
 *          ||   XX   |   XX   |   XX   |   XX   |   43   |   42   |   41   |   40   |
 *      5   ||--------Start of SV stream---------|--------Start of Window stream-----|
 * </pre>
 *
 * @author Lee Rhodes
 * @author Kevin Lang
 */
final class PreambleUtil {

  private PreambleUtil() {}

  static {
    if (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN) {
      throw new SketchesStateException("This sketch will not work on Big Endian CPUs.");
    }
  }

  private static final String LS = System.getProperty("line.separator");
  private static final String fmt = "%10d%10x";

  /**
   *  The serialization version for the set of serialization formats defined in this class
   */
  static final byte SER_VER = 1;

  //Flag bit masks, Byte 5
  static final int BIG_ENDIAN_FLAG_MASK     = 1; //Reserved.
  static final int COMPRESSED_FLAG_MASK     = 2;
  static final int HIP_FLAG_MASK            = 4;
  static final int SUP_VAL_FLAG_MASK        = 8; //num Suprising Values > 0
  static final int WINDOW_FLAG_MASK         = 16;//window length > 0

  //PREAMBLE SIZE

  /**
   * This defines the preamble space required by each of the formats in units of 4-byte integers.
   */
  private static final byte[] preIntDefs = { 2, 2, 4, 8, 4, 8, 6, 10 };

  /**
   * Returns the defined size of the preamble in units of integers (4 bytes) given the
   * <i>Format</i>.
   * @param format the given <i>Format</i>.
   * @return the defined size of the preamble in units of integers (4 bytes) given the <i>Format</i>.
   */
  static byte getDefinedPreInts(final Format format) {
    return preIntDefs[format.ordinal()];
  }

  //PREAMBLE LO_FIELD DEFINITIONS, OFFSETS, AND GETS

  /**
   * This defines the seven fields of the first eight bytes of the preamble.
   * The ordinal of these values defines the byte offset.
   * Do not change the order.
   */
  enum LoField { PRE_INTS, SER_VERSION, FAMILY, LG_K, FI_COL, FLAGS, SEED_HASH }

  /**
   * Returns the defined byte offset from the start of the preamble given a <i>LoField</i>.
   * This only applies to the first 8 bytes of the preamble.
   * @param loField the given <i>LoField</i>.
   * @return the defined byte offset from the start of the preamble given a <i>LoField</i>.
   */
  static int getLoFieldOffset(final LoField loField) {
    return loField.ordinal();
  }

  static int getPreInts(final Memory mem) {
    return mem.getByte(getLoFieldOffset(LoField.PRE_INTS)) & 0XFF;
  }

  static int getSerVer(final Memory mem) {
    return mem.getByte(getLoFieldOffset(LoField.SER_VERSION)) & 0XFF;
  }

  static Family getFamily(final Memory mem) {
    final int fam = mem.getByte(getLoFieldOffset(LoField.FAMILY)) & 0XFF;
    return Family.idToFamily(fam);
  }

  static int getLgK(final Memory mem) {
    return mem.getByte(getLoFieldOffset(LoField.LG_K)) & 0XFF;
  }

  static int getFiCol(final Memory mem) {
    return mem.getByte(getLoFieldOffset(LoField.FI_COL)) & 0XFF;
  }

  static int getFlags(final Memory mem) {
    return mem.getByte(getLoFieldOffset(LoField.FLAGS)) & 0XFF;
  }

  static short getSeedHash(final Memory mem) {
    return mem.getShort(getLoFieldOffset(LoField.SEED_HASH));
  }

  static int getFormatOrdinal(final Memory mem) {
    final int flags = getFlags(mem);
    return (flags >>> 2) & 0x7;
  }

  static Format getFormat(final Memory mem) {
    final int ordinal = getFormatOrdinal(mem);
    return Format.ordinalToFormat(ordinal);
  }

  static boolean hasHip(final Memory mem) {
    return (getFlags(mem) & HIP_FLAG_MASK) > 0;
  }

  static final boolean hasSv(final Memory mem) {
    return (getFlags(mem) & SUP_VAL_FLAG_MASK) > 0;
  }

  static final boolean hasWindow(final Memory mem) {
    return (getFlags(mem) & WINDOW_FLAG_MASK) > 0;
  }

  static final boolean isCompressed(final Memory mem) {
    return (getFlags(mem) & COMPRESSED_FLAG_MASK) > 0;
  }

  //PREAMBLE HI_FIELD DEFINITIONS

  /**
   * This defines the eight additional preamble fields located after the <i>LoField</i>.
   * Do not change the order.
   *
   * <p>Note: NUM_SV has dual meanings: In sparse and hybrid flavors it is equivalent to
   * numCoupons so it isn't stored separately. In pinned and sliding flavors is is the
   * numSV of the PairTable, which stores only surprising values.</p>
   */
  enum HiField { NUM_COUPONS, NUM_SV, KXP, HIP_ACCUM, SV_LENGTH_INTS, W_LENGTH_INTS, SV_STREAM,
    W_STREAM }

  //PREAMBLE HI_FIELD OFFSETS

  /**
   * This defines the byte offset for eac of the 8 <i>HiFields</i>
   * given the Format ordinal (1st dimension) and the HiField ordinal (2nd dimension).
   */
  private static final byte[][] hiFieldOffset = //[Format][HiField]
    { {0,  0,  0,  0,  0,  0,  0,  0},
      {0,  0,  0,  0,  0,  0,  0,  0},
      {8,  0,  0,  0, 12,  0, 16,  0},
      {8,  0, 16, 24, 12,  0, 32,  0},
      {8,  0,  0,  0,  0, 12,  0, 16},
      {8,  0, 16, 24,  0, 12,  0, 32},
      {8, 12,  0,  0, 16, 20, 24, 24}, //the 2nd 24 is not used.
      {8, 12, 16, 24, 32, 36, 40, 40}  //the 2nd 40 is not used.
    };

  /**
   * Returns the defined byte offset from the start of the preamble given the <i>HiField</i>
   * and the <i>Format</i>.
   * Note this can not be used to obtain the stream offsets.
   * @param format the desired <i>Format</i>
   * @param hiField the desired preamble <i>HiField</i> after the first eight bytes.
   * @return the defined byte offset from the start of the preamble for the given <i>HiField</i>
   * and the <i>Format</i>.
   */
  static long getHiFieldOffset(final Format format, final HiField hiField) {
    final int formatIdx = format.ordinal();
    final int hiFieldIdx = hiField.ordinal();
    final long fieldOffset = hiFieldOffset[formatIdx][hiFieldIdx] & 0xFF; //initially a byte
    if (fieldOffset == 0) {
      throw new SketchesStateException("Undefined preamble field given the Format: "
          + "Format: " + format.toString() + ", HiField: " + hiField.toString());
    }
    return fieldOffset;
  }

  //PREAMBLE HI_FIELD GETS

  static int getNumCoupons(final Memory mem) {
    final Format format = getFormat(mem);
    final HiField hiField = HiField.NUM_COUPONS;
    final long offset = getHiFieldOffset(format, hiField);
    return mem.getInt(offset);
  }

  static int getNumSv(final Memory mem) {
    final Format format = getFormat(mem);
    final HiField hiField = HiField.NUM_SV;
    final long offset = getHiFieldOffset(format, hiField);
    return mem.getInt(offset);
  }

  static int getSvLengthInts(final Memory mem) {
    final Format format = getFormat(mem);
    final HiField hiField = HiField.SV_LENGTH_INTS;
    final long offset = getHiFieldOffset(format, hiField);
    return mem.getInt(offset);
  }

  static int getWLengthInts(final Memory mem) {
    final Format format = getFormat(mem);
    final HiField hiField = HiField.W_LENGTH_INTS;
    final long offset = getHiFieldOffset(format, hiField);
    return mem.getInt(offset);
  }

  static double getKxP(final Memory mem) {
    final Format format = getFormat(mem);
    final HiField hiField = HiField.KXP;
    final long offset = getHiFieldOffset(format, hiField);
    return mem.getDouble(offset);
  }

  static double getHipAccum(final Memory mem) {
    final Format format = getFormat(mem);
    final HiField hiField = HiField.HIP_ACCUM;
    final long offset = getHiFieldOffset(format, hiField);
    return mem.getDouble(offset);
  }

  static long getSvStreamOffset(final Memory mem) {
    final Format format = getFormat(mem);
    final HiField svLenField = HiField.SV_LENGTH_INTS;
    if (!hasSv(mem)) {
      fieldError(format, svLenField);
    } else {
      final long svLengthInts = mem.getInt(getHiFieldOffset(format, HiField.SV_LENGTH_INTS)) & 0XFFFF_FFFFL;
      if (svLengthInts == 0) {
        throw new SketchesStateException("svLengthInts cannot be zero");
      }
    }
    long wLengthInts = 0;
    if (hasWindow(mem)) {
      wLengthInts = mem.getInt(getHiFieldOffset(format, HiField.W_LENGTH_INTS)) & 0XFFFF_FFFFL;
      if (wLengthInts == 0) {
        throw new SketchesStateException("wLengthInts cannot be zero");
      }
    }
    return (getPreInts(mem) + wLengthInts) << 2;
  }

  static long getWStreamOffset(final Memory mem) {
    final Format format = getFormat(mem);
    final HiField wLenField = HiField.W_LENGTH_INTS;
    if (!hasWindow(mem))  { fieldError(format, wLenField); }
    final long wLengthInts = mem.getInt(getHiFieldOffset(format, HiField.W_LENGTH_INTS)) & 0XFFFF_FFFFL;
      if (wLengthInts == 0) {
        throw new SketchesStateException("wLengthInts cannot be zero");
      }
    return getPreInts(mem) << 2;
  }

  static int[] getSvStream(final Memory mem) {
    final long offset = getSvStreamOffset(mem);
    final int svLengthInts = getSvLengthInts(mem);
    final int[] svStream = new int[svLengthInts];
    mem.getIntArray(offset, svStream, 0, svLengthInts);
    return svStream;
  }

  static int[] getWStream(final Memory mem) {
    final long offset = getWStreamOffset(mem);
    final int wLength = getWLengthInts(mem);
    final int[] wStream = new int[wLength];
    mem.getIntArray(offset, wStream, 0, wLength);
    return wStream;
  }

  // PUT INTO MEMORY

  static void putEmptyMerged(final WritableMemory wmem,
      final int lgK,
      final short seedHash) {
    final Format format = Format.EMPTY_MERGED;
    final byte preInts = getDefinedPreInts(format);
    final byte fiCol = (byte) 0;
    final byte flags = (byte) ((format.ordinal() << 2) | COMPRESSED_FLAG_MASK);
    checkCapacity(wmem.getCapacity(), 8);
    putFirst8(wmem, preInts, (byte) lgK, fiCol, flags, seedHash);
  }

  static void putEmptyHip(final WritableMemory wmem,
      final int lgK,
      final short seedHash) {
    final Format format = Format.EMPTY_HIP;
    final byte preInts = getDefinedPreInts(format);
    final byte fiCol = (byte) 0;
    final byte flags = (byte) ((format.ordinal() << 2) | COMPRESSED_FLAG_MASK);
    checkCapacity(wmem.getCapacity(), 8);
    putFirst8(wmem, preInts, (byte) lgK, fiCol, flags, seedHash);
  }

  static void putSparseHybridMerged(final WritableMemory wmem,
      final int lgK,
      final int numCoupons, //unsigned
      final int svLengthInts,
      final short seedHash,
      final int[] svStream) {
    final Format format = Format.SPARSE_HYBRID_MERGED;
    final byte preInts = getDefinedPreInts(format);
    final byte fiCol = (byte) 0;
    final byte flags = (byte) ((format.ordinal() << 2) | COMPRESSED_FLAG_MASK);
    checkCapacity(wmem.getCapacity(), 4L * (preInts + svLengthInts));
    putFirst8(wmem, preInts, (byte) lgK, fiCol, flags, seedHash);

    wmem.putInt(getHiFieldOffset(format, HiField.NUM_COUPONS), numCoupons);
    wmem.putInt(getHiFieldOffset(format, HiField.SV_LENGTH_INTS), svLengthInts);
    wmem.putIntArray(getSvStreamOffset(wmem), svStream, 0, svLengthInts);
  }

  static void putSparseHybridHip(final WritableMemory wmem,
      final int lgK,
      final int numCoupons, //unsigned
      final int svLengthInts,
      final double kxp,
      final double hipAccum,
      final short seedHash,
      final int[] svStream) {
    final Format format = Format.SPARSE_HYBRID_HIP;
    final byte preInts = getDefinedPreInts(format);
    final byte fiCol = (byte) 0;
    final byte flags = (byte) ((format.ordinal() << 2) | COMPRESSED_FLAG_MASK);
    checkCapacity(wmem.getCapacity(), 4L * (preInts + svLengthInts));
    putFirst8(wmem, preInts, (byte) lgK, fiCol, flags, seedHash);

    wmem.putInt(getHiFieldOffset(format, HiField.NUM_COUPONS), numCoupons);
    wmem.putInt(getHiFieldOffset(format, HiField.SV_LENGTH_INTS), svLengthInts);
    wmem.putDouble(getHiFieldOffset(format, HiField.KXP), kxp);
    wmem.putDouble(getHiFieldOffset(format, HiField.HIP_ACCUM), hipAccum);
    wmem.putIntArray(getSvStreamOffset(wmem), svStream, 0, svLengthInts);
  }

  static void putPinnedSlidingMergedNoSv(final WritableMemory wmem,
      final int lgK,
      final int fiCol,
      final int numCoupons, //unsigned
      final int wLengthInts,
      final short seedHash,
      final int[] wStream) {
    final Format format = Format.PINNED_SLIDING_MERGED_NOSV;
    final byte preInts = getDefinedPreInts(format);
    final byte flags = (byte) ((format.ordinal() << 2) | COMPRESSED_FLAG_MASK);
    checkCapacity(wmem.getCapacity(), 4L * (preInts + wLengthInts));
    putFirst8(wmem, preInts, (byte) lgK, (byte) fiCol, flags, seedHash);

    wmem.putInt(getHiFieldOffset(format, HiField.NUM_COUPONS), numCoupons);
    wmem.putInt(getHiFieldOffset(format, HiField.W_LENGTH_INTS), wLengthInts);
    wmem.putIntArray(getWStreamOffset(wmem), wStream, 0, wLengthInts);
  }

  static void putPinnedSlidingHipNoSv(final WritableMemory wmem,
      final int lgK,
      final int fiCol,
      final int numCoupons, //unsigned
      final int wLengthInts,
      final double kxp,
      final double hipAccum,
      final short seedHash,
      final int[] wStream) {
    final Format format = Format.PINNED_SLIDING_HIP_NOSV;
    final byte preInts = getDefinedPreInts(format);
    final byte flags = (byte) ((format.ordinal() << 2) | COMPRESSED_FLAG_MASK);
    checkCapacity(wmem.getCapacity(), 4L * (preInts + wLengthInts));
    putFirst8(wmem, preInts, (byte) lgK, (byte) fiCol, flags, seedHash);

    wmem.putInt(getHiFieldOffset(format, HiField.NUM_COUPONS), numCoupons);
    wmem.putInt(getHiFieldOffset(format, HiField.W_LENGTH_INTS), wLengthInts);
    wmem.putDouble(getHiFieldOffset(format, HiField.KXP), kxp);
    wmem.putDouble(getHiFieldOffset(format, HiField.HIP_ACCUM), hipAccum);
    wmem.putIntArray(getWStreamOffset(wmem), wStream, 0, wLengthInts);
  }

  static void putPinnedSlidingMerged(final WritableMemory wmem,
      final int lgK,
      final int fiCol,
      final int numCoupons, //unsigned
      final int numSv,
      final int svLengthInts,
      final int wLengthInts,
      final short seedHash,
      final int[] svStream,
      final int[] wStream) {
    final Format format = Format.PINNED_SLIDING_MERGED;
    final byte preInts = getDefinedPreInts(format);
    final byte flags = (byte) ((format.ordinal() << 2) | COMPRESSED_FLAG_MASK);
    checkCapacity(wmem.getCapacity(), 4L * (preInts + svLengthInts + wLengthInts));
    putFirst8(wmem, preInts, (byte) lgK, (byte) fiCol, flags, seedHash);

    wmem.putInt(getHiFieldOffset(format, HiField.NUM_COUPONS), numCoupons);
    wmem.putInt(getHiFieldOffset(format, HiField.NUM_SV), numSv);
    wmem.putInt(getHiFieldOffset(format, HiField.SV_LENGTH_INTS), svLengthInts);
    wmem.putInt(getHiFieldOffset(format, HiField.W_LENGTH_INTS), wLengthInts);
    wmem.putIntArray(getSvStreamOffset(wmem), svStream, 0, svLengthInts);
    wmem.putIntArray(getWStreamOffset(wmem), wStream, 0, wLengthInts);
  }

  static void putPinnedSlidingHip(final WritableMemory wmem,
      final int lgK,
      final int fiCol,
      final int numCoupons, //unsigned
      final int numSv,
      final double kxp,
      final double hipAccum,
      final int svLengthInts,
      final int wLengthInts,
      final short seedHash,
      final int[] svStream,
      final int[] wStream) {
    final Format format = Format.PINNED_SLIDING_HIP;
    final byte preInts = getDefinedPreInts(format);
    final byte flags = (byte) ((format.ordinal() << 2) | COMPRESSED_FLAG_MASK);
    checkCapacity(wmem.getCapacity(), 4L * (preInts + svLengthInts + wLengthInts));
    putFirst8(wmem, preInts, (byte) lgK, (byte) fiCol, flags, seedHash);

    wmem.putInt(getHiFieldOffset(format, HiField.NUM_COUPONS), numCoupons);
    wmem.putInt(getHiFieldOffset(format, HiField.NUM_SV), numSv);
    wmem.putDouble(getHiFieldOffset(format, HiField.KXP), kxp);
    wmem.putDouble(getHiFieldOffset(format, HiField.HIP_ACCUM), hipAccum);
    wmem.putInt(getHiFieldOffset(format, HiField.SV_LENGTH_INTS), svLengthInts);
    wmem.putInt(getHiFieldOffset(format, HiField.W_LENGTH_INTS), wLengthInts);
    wmem.putIntArray(getSvStreamOffset(wmem), svStream, 0, svLengthInts);
    wmem.putIntArray(getWStreamOffset(wmem), wStream, 0, wLengthInts);
  }

  private static void putFirst8(final WritableMemory wmem, final byte preInts, final byte lgK,
      final byte fiCol, final byte flags, final short seedHash) {
    wmem.clear(0L, 4L * preInts);
    wmem.putByte(getLoFieldOffset(LoField.PRE_INTS), preInts);
    wmem.putByte(getLoFieldOffset(LoField.SER_VERSION), SER_VER);
    wmem.putByte(getLoFieldOffset(LoField.FAMILY), (byte) Family.CPC.getID());
    wmem.putByte(getLoFieldOffset(LoField.LG_K), lgK);
    wmem.putByte(getLoFieldOffset(LoField.FI_COL), fiCol);
    wmem.putByte(getLoFieldOffset(LoField.FLAGS), flags);
    wmem.putShort(getLoFieldOffset(LoField.SEED_HASH), seedHash);
  }

  //TO STRING

  static String toString(final byte[] byteArr, final boolean detail) {
    final Memory mem = Memory.wrap(byteArr);
    return toString(mem, detail);
  }

  static String toString(final Memory mem, final boolean detail) {
    final long capBytes = mem.getCapacity();
    //Lo Fields Preamble, first 7 fields, first 8 bytes
    final int preInts = mem.getByte(getLoFieldOffset(LoField.PRE_INTS)) & 0xFF;
    final int serVer = mem.getByte(getLoFieldOffset(LoField.SER_VERSION)) & 0xFF;
    final Family family = Family.idToFamily(mem.getByte(getLoFieldOffset(LoField.FAMILY)) & 0xFF);
    final int lgK = mem.getByte(getLoFieldOffset(LoField.LG_K)) & 0xFF;
    final int fiCol = mem.getByte(getLoFieldOffset(LoField.FI_COL)) & 0xFF;
    final int flags = mem.getByte(getLoFieldOffset(LoField.FLAGS)) & 0XFF;
    final int seedHash = mem.getShort(getLoFieldOffset(LoField.SEED_HASH)) & 0XFFFF;
    final String seedHashStr = Integer.toHexString(seedHash);

    //Flags of the Flags byte
    final String flagsStr = zeroPad(Integer.toBinaryString(flags), 8) + ", " + (flags);
    final boolean bigEndian = (flags & BIG_ENDIAN_FLAG_MASK) > 0;
    final boolean compressed = (flags & COMPRESSED_FLAG_MASK) > 0;
    final boolean hasHip = (flags & HIP_FLAG_MASK) > 0;
    final boolean hasSV = (flags & SUP_VAL_FLAG_MASK) > 0;
    final boolean hasWindow = (flags & WINDOW_FLAG_MASK) > 0;

    final int formatOrdinal = (flags >>> 2) & 0x7;
    final Format format = Format.ordinalToFormat(formatOrdinal);

    final String nativeOrderStr = ByteOrder.nativeOrder().toString();

    long numCoupons = 0;
    long numSv = 0;
    long winOffset = 0;
    long svLengthInts = 0;
    long wLengthInts = 0;
    double kxp = 0;
    double hipAccum = 0;
    long svStreamStart = 0;
    long wStreamStart = 0;
    long reqBytes = 0;

    final StringBuilder sb = new StringBuilder();
    sb.append(LS);
    sb.append("### CPC SKETCH IMAGE - PREAMBLE:").append(LS);
    sb.append("Format                          : ").append(format.name()).append(LS);
    sb.append("Byte 0: Preamble Ints           : ").append(preInts).append(LS);
    sb.append("Byte 1: SerVer                  : ").append(serVer).append(LS);
    sb.append("Byte 2: Family                  : ").append(family).append(LS);
    sb.append("Byte 3: lgK                     : ").append(lgK).append(LS);
    sb.append("Byte 4: First Interesting Col   : ").append(fiCol).append(LS);
    sb.append("Byte 5: Flags                   : ").append(flagsStr).append(LS);
    sb.append("  BIG_ENDIAN_STORAGE            : ").append(bigEndian).append(LS);
    sb.append("  (Native Byte Order)           : ").append(nativeOrderStr).append(LS);
    sb.append("  Compressed                    : ").append(compressed).append(LS);
    sb.append("  Has HIP                       : ").append(hasHip).append(LS);
    sb.append("  Has Surprising Values         : ").append(hasSV).append(LS);
    sb.append("  Has Window Values             : ").append(hasWindow).append(LS);
    sb.append("Byte 6, 7: Seed Hash            : ").append(seedHashStr).append(LS);

    final Flavor flavor;

    switch (format) {
      case EMPTY_MERGED :
      case EMPTY_HIP : {
        flavor = CpcUtil.determineFlavor(lgK, numCoupons);
        sb.append("Flavor                          : ").append(flavor).append(LS);
        break;
      }
      case SPARSE_HYBRID_MERGED : {
        numCoupons = mem.getInt(getHiFieldOffset(format, HiField.NUM_COUPONS)) & 0xFFFF_FFFFL;

        numSv = numCoupons;
        svLengthInts = mem.getInt(getHiFieldOffset(format, HiField.SV_LENGTH_INTS)) & 0xFFFF_FFFFL;
        svStreamStart = getSvStreamOffset(mem);
        reqBytes = svStreamStart + (svLengthInts << 2);
        flavor = CpcUtil.determineFlavor(lgK, numCoupons);
        sb.append("Flavor                          : ").append(flavor).append(LS);
        sb.append("Num Coupons                     : ").append(numCoupons).append(LS);

        sb.append("Num SV                          : ").append(numSv).append(LS);
        sb.append("SV Length Ints                  : ").append(svLengthInts).append(LS);
        sb.append("SV Stream Start                 : ").append(svStreamStart).append(LS);
        break;
      }
      case SPARSE_HYBRID_HIP : {
        numCoupons = mem.getInt(getHiFieldOffset(format, HiField.NUM_COUPONS)) & 0xFFFF_FFFFL;

        numSv = numCoupons;
        svLengthInts = mem.getInt(getHiFieldOffset(format, HiField.SV_LENGTH_INTS)) & 0xFFFF_FFFFL;
        svStreamStart = getSvStreamOffset(mem);

        kxp = mem.getDouble(getHiFieldOffset(format, HiField.KXP));
        hipAccum = mem.getDouble(getHiFieldOffset(format, HiField.HIP_ACCUM));
        reqBytes = svStreamStart + (svLengthInts << 2);

        flavor = CpcUtil.determineFlavor(lgK, numCoupons);
        sb.append("Flavor                          : ").append(flavor).append(LS);
        sb.append("Num Coupons                     : ").append(numCoupons).append(LS);

        sb.append("Num SV                          : ").append(numSv).append(LS);
        sb.append("SV Length Ints                  : ").append(svLengthInts).append(LS);
        sb.append("SV Stream Start                 : ").append(svStreamStart).append(LS);

        sb.append("KxP                             : ").append(kxp).append(LS);
        sb.append("HipAccum                        : ").append(hipAccum).append(LS);
        break;
      }
      case PINNED_SLIDING_MERGED_NOSV : {
        numCoupons = mem.getInt(getHiFieldOffset(format, HiField.NUM_COUPONS)) & 0xFFFF_FFFFL;

        winOffset = CpcUtil.determineCorrectOffset(lgK, numCoupons);
        wLengthInts = mem.getInt(getHiFieldOffset(format, HiField.W_LENGTH_INTS)) & 0xFFFF_FFFFL;
        wStreamStart = getWStreamOffset(mem);
        reqBytes = wStreamStart + (wLengthInts << 2);

        flavor = CpcUtil.determineFlavor(lgK, numCoupons);
        sb.append("Flavor                          : ").append(flavor).append(LS);
        sb.append("Num Coupons                     : ").append(numCoupons).append(LS);

        sb.append("Window Offset                   : ").append(winOffset).append(LS);
        sb.append("Window Length Ints              : ").append(wLengthInts).append(LS);
        sb.append("Window Stream Start             : ").append(wStreamStart).append(LS);
        break;
      }
      case PINNED_SLIDING_HIP_NOSV : {
        numCoupons = mem.getInt(getHiFieldOffset(format, HiField.NUM_COUPONS)) & 0xFFFF_FFFFL;

        winOffset = CpcUtil.determineCorrectOffset(lgK, numCoupons);
        wLengthInts = mem.getInt(getHiFieldOffset(format, HiField.W_LENGTH_INTS)) & 0xFFFF_FFFFL;
        wStreamStart = getWStreamOffset(mem);

        kxp = mem.getDouble(getHiFieldOffset(format, HiField.KXP));
        hipAccum = mem.getDouble(getHiFieldOffset(format, HiField.HIP_ACCUM));
        reqBytes = wStreamStart + (wLengthInts << 2);

        flavor = CpcUtil.determineFlavor(lgK, numCoupons);
        sb.append("Flavor                          : ").append(flavor).append(LS);
        sb.append("Num Coupons                     : ").append(numCoupons).append(LS);

        sb.append("Window Offset                   : ").append(winOffset).append(LS);
        sb.append("Window Length Ints              : ").append(wLengthInts).append(LS);
        sb.append("Window Stream Start             : ").append(wStreamStart).append(LS);

        sb.append("KxP                             : ").append(kxp).append(LS);
        sb.append("HipAccum                        : ").append(hipAccum).append(LS);
        break;
      }
      case PINNED_SLIDING_MERGED : {
        numCoupons = mem.getInt(getHiFieldOffset(format, HiField.NUM_COUPONS) & 0xFFFF_FFFFL);

        winOffset = CpcUtil.determineCorrectOffset(lgK, numCoupons);
        wLengthInts = mem.getInt(getHiFieldOffset(format, HiField.W_LENGTH_INTS)) & 0xFFFF_FFFFL;

        numSv = mem.getInt(getHiFieldOffset(format, HiField.NUM_SV)) & 0xFFFF_FFFFL;
        svLengthInts = mem.getInt(getHiFieldOffset(format, HiField.SV_LENGTH_INTS)) & 0xFFFF_FFFFL;

        wStreamStart = getWStreamOffset(mem);
        svStreamStart = getSvStreamOffset(mem);
        reqBytes = svStreamStart + (svLengthInts << 2);

        flavor = CpcUtil.determineFlavor(lgK, numCoupons);
        sb.append("Flavor                          : ").append(flavor).append(LS);
        sb.append("Num Coupons                     : ").append(numCoupons).append(LS);

        sb.append("Num SV                          : ").append(numSv).append(LS);
        sb.append("SV Length Ints                  : ").append(svLengthInts).append(LS);
        sb.append("SV Stream Start                 : ").append(svStreamStart).append(LS);

        sb.append("Window Offset                   : ").append(winOffset).append(LS);
        sb.append("Window Length Ints              : ").append(wLengthInts).append(LS);
        sb.append("Window Stream Start             : ").append(wStreamStart).append(LS);
        break;
      }
      case PINNED_SLIDING_HIP : {
        numCoupons = mem.getInt(getHiFieldOffset(format, HiField.NUM_COUPONS) & 0xFFFF_FFFFL);

        winOffset = CpcUtil.determineCorrectOffset(lgK, numCoupons);
        wLengthInts = mem.getInt(getHiFieldOffset(format, HiField.W_LENGTH_INTS)) & 0xFFFF_FFFFL;

        numSv = mem.getInt(getHiFieldOffset(format, HiField.NUM_SV)) & 0xFFFF_FFFFL;
        svLengthInts = mem.getInt(getHiFieldOffset(format, HiField.SV_LENGTH_INTS)) & 0xFFFF_FFFFL;
        wStreamStart = getWStreamOffset(mem);
        svStreamStart = getSvStreamOffset(mem);

        kxp = mem.getDouble(getHiFieldOffset(format, HiField.KXP));
        hipAccum = mem.getDouble(getHiFieldOffset(format, HiField.HIP_ACCUM));
        reqBytes = svStreamStart + (svLengthInts << 2);

        flavor = CpcUtil.determineFlavor(lgK, numCoupons);
        sb.append("Flavor                          : ").append(flavor).append(LS);
        sb.append("Num Coupons                     : ").append(numCoupons).append(LS);

        sb.append("Num SV                          : ").append(numSv).append(LS);
        sb.append("SV Length Ints                  : ").append(svLengthInts).append(LS);
        sb.append("SV Stream Start                 : ").append(svStreamStart).append(LS);

        sb.append("Window Offset                   : ").append(winOffset).append(LS);
        sb.append("Window Length Ints              : ").append(wLengthInts).append(LS);
        sb.append("Window Stream Start             : ").append(wStreamStart).append(LS);

        sb.append("KxP                             : ").append(kxp).append(LS);
        sb.append("HipAccum                        : ").append(hipAccum).append(LS);
        break;
      }
    }

    sb.append("Actual Bytes                    : ").append(capBytes).append(LS);
    sb.append("Required Bytes                  : ").append(reqBytes).append(LS);

    if (detail) {
      sb.append(LS).append("### CPC SKETCH IMAGE - DATA").append(LS);
      if (wLengthInts > 0) {
        sb.append(LS).append("Window Stream:").append(LS);
        listData(mem, wStreamStart, wLengthInts, sb);
      }
      if (svLengthInts > 0) {
        sb.append(LS).append("SV Stream:").append(LS);
        listData(mem, svStreamStart, svLengthInts, sb);
      }
    }
    sb.append("### END CPC SKETCH IMAGE").append(LS);
    return sb.toString();
  } //end toString(mem)

  private static void listData(final Memory mem, final long offsetBytes, final long lengthInts,
      final StringBuilder sb) {
    final long memCap = mem.getCapacity();
    final long expectedCap = offsetBytes + (4L * lengthInts);
    checkCapacity(memCap, expectedCap);
    for (int i = 0; i < lengthInts; i++) {
      sb.append(String.format(fmt, i, mem.getInt(offsetBytes + (4L * i)))).append(LS);
    }
  }

  static void fieldError(final Format format, final HiField hiField) {
    throw new SketchesArgumentException(
        "Operation is illegal: Format = " + format.name() + ", HiField = " + hiField);
  }

  static void checkCapacity(final long memCap, final long expectedCap) {
    if (memCap < expectedCap) {
      throw new SketchesArgumentException(
          "Insufficient Image Bytes = " + memCap + ", Expected = " + expectedCap);
    }
  }

  //basic checks of SerVer, Format, preInts, Family, fiCol, lgK.
  static void checkLoPreamble(final Memory mem) {
    rtAssertEquals(getSerVer(mem), SER_VER & 0XFF);
    final Format fmt = getFormat(mem);
    final int preIntsDef = getDefinedPreInts(fmt) & 0XFF;
    rtAssertEquals(getPreInts(mem), preIntsDef);
    final Family fam = getFamily(mem);
    rtAssert(fam == Family.CPC);
    final int lgK = getLgK(mem);
    rtAssert((lgK >= 4) && (lgK <= 26));
    final int fiCol = getFiCol(mem);
    rtAssert((fiCol <= 63) && (fiCol >= 0));
  }

}
//@formatter:on