ARROW-11839: [C++] Use xsimd for generation of accelerated bit-unpacking
The custom per-ISA code generation scripts (AVX2, AVX512) are replaced with a single code generation script that outputs xsimd code for any SIMD bit-width, in an ISA-agnostic way.
Also add a Neon optimized version of bit-unpacking that leverages the generated code for 128-bit SIMD.
Closes #9614 from pitrou/ARROW-11839-xsimd-bpacking
Authored-by: Antoine Pitrou <antoine@python.org>
Signed-off-by: Antoine Pitrou <antoine@python.org>
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 9f68c56..6e25955 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -452,6 +452,7 @@
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} ${ARROW_ARMV8_ARCH_FLAG}")
if(NOT ARROW_SIMD_LEVEL STREQUAL "NONE")
+ set(ARROW_HAVE_NEON ON)
add_definitions(-DARROW_HAVE_NEON)
endif()
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index df72dcc..3623283 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -228,6 +228,9 @@
set_source_files_properties(util/bpacking_avx512.cc PROPERTIES COMPILE_FLAGS
${ARROW_AVX512_FLAG})
endif()
+if(ARROW_HAVE_NEON)
+ list(APPEND ARROW_SRCS util/bpacking_neon.cc)
+endif()
if(APPLE)
list(APPEND ARROW_SRCS vendored/datetime/ios.mm)
diff --git a/cpp/src/arrow/util/bpacking.cc b/cpp/src/arrow/util/bpacking.cc
index 0263475..2e658fd 100644
--- a/cpp/src/arrow/util/bpacking.cc
+++ b/cpp/src/arrow/util/bpacking.cc
@@ -27,6 +27,9 @@
#if defined(ARROW_HAVE_RUNTIME_AVX512)
#include "arrow/util/bpacking_avx512.h"
#endif
+#if defined(ARROW_HAVE_NEON)
+#include "arrow/util/bpacking_neon.h"
+#endif
namespace arrow {
namespace internal {
@@ -163,8 +166,12 @@
} // namespace
int unpack32(const uint32_t* in, uint32_t* out, int batch_size, int num_bits) {
+#if defined(ARROW_HAVE_NEON)
+ return unpack32_neon(in, out, batch_size, num_bits);
+#else
static DynamicDispatch<Unpack32DynamicFunction> dispatch;
return dispatch.func(in, out, batch_size, num_bits);
+#endif
}
} // namespace internal
diff --git a/cpp/src/arrow/util/bpacking_avx2.cc b/cpp/src/arrow/util/bpacking_avx2.cc
index 63b914b..5a3a7ba 100644
--- a/cpp/src/arrow/util/bpacking_avx2.cc
+++ b/cpp/src/arrow/util/bpacking_avx2.cc
@@ -16,121 +16,15 @@
// under the License.
#include "arrow/util/bpacking_avx2.h"
-#include "arrow/util/bpacking_avx2_generated.h"
-#include "arrow/util/logging.h"
+#include "arrow/util/bpacking_simd256_generated.h"
+#include "arrow/util/bpacking_simd_internal.h"
namespace arrow {
namespace internal {
int unpack32_avx2(const uint32_t* in, uint32_t* out, int batch_size, int num_bits) {
- batch_size = batch_size / 32 * 32;
- int num_loops = batch_size / 32;
-
- switch (num_bits) {
- case 0:
- for (int i = 0; i < num_loops; ++i) in = unpack0_32_avx2(in, out + i * 32);
- break;
- case 1:
- for (int i = 0; i < num_loops; ++i) in = unpack1_32_avx2(in, out + i * 32);
- break;
- case 2:
- for (int i = 0; i < num_loops; ++i) in = unpack2_32_avx2(in, out + i * 32);
- break;
- case 3:
- for (int i = 0; i < num_loops; ++i) in = unpack3_32_avx2(in, out + i * 32);
- break;
- case 4:
- for (int i = 0; i < num_loops; ++i) in = unpack4_32_avx2(in, out + i * 32);
- break;
- case 5:
- for (int i = 0; i < num_loops; ++i) in = unpack5_32_avx2(in, out + i * 32);
- break;
- case 6:
- for (int i = 0; i < num_loops; ++i) in = unpack6_32_avx2(in, out + i * 32);
- break;
- case 7:
- for (int i = 0; i < num_loops; ++i) in = unpack7_32_avx2(in, out + i * 32);
- break;
- case 8:
- for (int i = 0; i < num_loops; ++i) in = unpack8_32_avx2(in, out + i * 32);
- break;
- case 9:
- for (int i = 0; i < num_loops; ++i) in = unpack9_32_avx2(in, out + i * 32);
- break;
- case 10:
- for (int i = 0; i < num_loops; ++i) in = unpack10_32_avx2(in, out + i * 32);
- break;
- case 11:
- for (int i = 0; i < num_loops; ++i) in = unpack11_32_avx2(in, out + i * 32);
- break;
- case 12:
- for (int i = 0; i < num_loops; ++i) in = unpack12_32_avx2(in, out + i * 32);
- break;
- case 13:
- for (int i = 0; i < num_loops; ++i) in = unpack13_32_avx2(in, out + i * 32);
- break;
- case 14:
- for (int i = 0; i < num_loops; ++i) in = unpack14_32_avx2(in, out + i * 32);
- break;
- case 15:
- for (int i = 0; i < num_loops; ++i) in = unpack15_32_avx2(in, out + i * 32);
- break;
- case 16:
- for (int i = 0; i < num_loops; ++i) in = unpack16_32_avx2(in, out + i * 32);
- break;
- case 17:
- for (int i = 0; i < num_loops; ++i) in = unpack17_32_avx2(in, out + i * 32);
- break;
- case 18:
- for (int i = 0; i < num_loops; ++i) in = unpack18_32_avx2(in, out + i * 32);
- break;
- case 19:
- for (int i = 0; i < num_loops; ++i) in = unpack19_32_avx2(in, out + i * 32);
- break;
- case 20:
- for (int i = 0; i < num_loops; ++i) in = unpack20_32_avx2(in, out + i * 32);
- break;
- case 21:
- for (int i = 0; i < num_loops; ++i) in = unpack21_32_avx2(in, out + i * 32);
- break;
- case 22:
- for (int i = 0; i < num_loops; ++i) in = unpack22_32_avx2(in, out + i * 32);
- break;
- case 23:
- for (int i = 0; i < num_loops; ++i) in = unpack23_32_avx2(in, out + i * 32);
- break;
- case 24:
- for (int i = 0; i < num_loops; ++i) in = unpack24_32_avx2(in, out + i * 32);
- break;
- case 25:
- for (int i = 0; i < num_loops; ++i) in = unpack25_32_avx2(in, out + i * 32);
- break;
- case 26:
- for (int i = 0; i < num_loops; ++i) in = unpack26_32_avx2(in, out + i * 32);
- break;
- case 27:
- for (int i = 0; i < num_loops; ++i) in = unpack27_32_avx2(in, out + i * 32);
- break;
- case 28:
- for (int i = 0; i < num_loops; ++i) in = unpack28_32_avx2(in, out + i * 32);
- break;
- case 29:
- for (int i = 0; i < num_loops; ++i) in = unpack29_32_avx2(in, out + i * 32);
- break;
- case 30:
- for (int i = 0; i < num_loops; ++i) in = unpack30_32_avx2(in, out + i * 32);
- break;
- case 31:
- for (int i = 0; i < num_loops; ++i) in = unpack31_32_avx2(in, out + i * 32);
- break;
- case 32:
- for (int i = 0; i < num_loops; ++i) in = unpack32_32_avx2(in, out + i * 32);
- break;
- default:
- DCHECK(false) << "Unsupported num_bits";
- }
-
- return batch_size;
+ return unpack32_specialized<UnpackBits256<DispatchLevel::AVX2>>(in, out, batch_size,
+ num_bits);
}
} // namespace internal
diff --git a/cpp/src/arrow/util/bpacking_avx2_codegen.py b/cpp/src/arrow/util/bpacking_avx2_codegen.py
deleted file mode 100644
index e60aed8..0000000
--- a/cpp/src/arrow/util/bpacking_avx2_codegen.py
+++ /dev/null
@@ -1,203 +0,0 @@
-#!/bin/python
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Usage: python bpacking_avx2_codegen.py > bpacking_avx2_generated.h
-
-
-def print_unpack_bit_func(bit):
- shift = 0
- shifts = []
- in_index = 0
- inls = []
- mask = (1 << bit) - 1
- bracket = "{"
-
- print(
- f"inline static const uint32_t* unpack{bit}_32_avx2(const uint32_t* in, uint32_t* out) {bracket}")
- print(" using ::arrow::util::SafeLoad;")
- print(" uint32_t mask = 0x%x;" % mask)
- print(" __m256i reg_shifts, reg_inls, reg_masks;")
- print(" __m256i results;")
-
- print("")
- for i in range(32):
- if shift + bit == 32:
- shifts.append(shift)
- inls.append(f"SafeLoad(in + {in_index})")
- in_index += 1
- shift = 0
- elif shift + bit > 32: # cross the boundary
- inls.append(
- f"SafeLoad(in + {in_index}) >> {shift} | SafeLoad(in + {in_index + 1}) << {32 - shift}")
- in_index += 1
- shift = bit - (32 - shift)
- shifts.append(0) # zero shift
- else:
- shifts.append(shift)
- inls.append(f"SafeLoad(in + {in_index})")
- shift += bit
-
- print(" reg_masks = _mm256_set1_epi32(mask);")
- print("")
-
- print(" // shift the first 8 outs")
- print(
- f" reg_shifts = _mm256_set_epi32({shifts[7]}, {shifts[6]}, {shifts[5]}, {shifts[4]},")
- print(
- f" {shifts[3]}, {shifts[2]}, {shifts[1]}, {shifts[0]});")
- print(f" reg_inls = _mm256_set_epi32({inls[7]}, {inls[6]},")
- print(f" {inls[5]}, {inls[4]},")
- print(f" {inls[3]}, {inls[2]},")
- print(f" {inls[1]}, {inls[0]});")
- print(
- " results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);")
- print(" _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);")
- print(" out += 8;")
- print("")
-
- print(" // shift the second 8 outs")
- print(
- f" reg_shifts = _mm256_set_epi32({shifts[15]}, {shifts[14]}, {shifts[13]}, {shifts[12]},")
- print(
- f" {shifts[11]}, {shifts[10]}, {shifts[9]}, {shifts[8]});")
- print(f" reg_inls = _mm256_set_epi32({inls[15]}, {inls[14]},")
- print(f" {inls[13]}, {inls[12]},")
- print(f" {inls[11]}, {inls[10]},")
- print(f" {inls[9]}, {inls[8]});")
- print(
- " results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);")
- print(" _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);")
- print(" out += 8;")
- print("")
-
- print(" // shift the third 8 outs")
- print(
- f" reg_shifts = _mm256_set_epi32({shifts[23]}, {shifts[22]}, {shifts[21]}, {shifts[20]},")
- print(
- f" {shifts[19]}, {shifts[18]}, {shifts[17]}, {shifts[16]});")
- print(f" reg_inls = _mm256_set_epi32({inls[23]}, {inls[22]},")
- print(f" {inls[21]}, {inls[20]},")
- print(f" {inls[19]}, {inls[18]},")
- print(f" {inls[17]}, {inls[16]});")
- print(
- " results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);")
- print(" _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);")
- print(" out += 8;")
- print("")
-
- print(" // shift the last 8 outs")
- print(
- f" reg_shifts = _mm256_set_epi32({shifts[31]}, {shifts[30]}, {shifts[29]}, {shifts[28]},")
- print(
- f" {shifts[27]}, {shifts[26]}, {shifts[25]}, {shifts[24]});")
- print(f" reg_inls = _mm256_set_epi32({inls[31]}, {inls[30]},")
- print(f" {inls[29]}, {inls[28]},")
- print(f" {inls[27]}, {inls[26]},")
- print(f" {inls[25]}, {inls[24]});")
- print(
- " results = _mm256_and_si256(_mm256_srlv_epi32(reg_inls, reg_shifts), reg_masks);")
- print(" _mm256_storeu_si256(reinterpret_cast<__m256i*>(out), results);")
- print(" out += 8;")
-
- print("")
- print(f" in += {bit};")
- print("")
- print(" return in;")
- print("}")
-
-
-def print_unpack_bit0_func():
- print(
- "inline static const uint32_t* unpack0_32_avx2(const uint32_t* in, uint32_t* out) {")
- print(" memset(out, 0x0, 32 * sizeof(*out));")
- print(" out += 32;")
- print("")
- print(" return in;")
- print("}")
-
-
-def print_unpack_bit32_func():
- print(
- "inline static const uint32_t* unpack32_32_avx2(const uint32_t* in, uint32_t* out) {")
- print(" memcpy(out, in, 32 * sizeof(*out));")
- print(" in += 32;")
- print(" out += 32;")
- print("")
- print(" return in;")
- print("}")
-
-
-def print_copyright():
- print(
- """// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.""")
-
-
-def print_note():
- print("//")
- print("// Automatically generated file; DO NOT EDIT.")
-
-
-def main():
- print_copyright()
- print_note()
- print("")
- print("#pragma once")
- print("")
- print("#include <stdint.h>")
- print("#include <string.h>")
- print("")
- print("#ifdef _MSC_VER")
- print("#include <intrin.h>")
- print("#else")
- print("#include <immintrin.h>")
- print("#endif")
- print("")
- print('#include "arrow/util/ubsan.h"')
- print("")
- print("namespace arrow {")
- print("namespace internal {")
- print("")
- print_unpack_bit0_func()
- print("")
- for i in range(1, 32):
- print_unpack_bit_func(i)
- print("")
- print_unpack_bit32_func()
- print("")
- print("} // namespace internal")
- print("} // namespace arrow")
-
-
-if __name__ == '__main__':
- main()
diff --git a/cpp/src/arrow/util/bpacking_avx512.cc b/cpp/src/arrow/util/bpacking_avx512.cc
index 98eb4d3..08ccd3f 100644
--- a/cpp/src/arrow/util/bpacking_avx512.cc
+++ b/cpp/src/arrow/util/bpacking_avx512.cc
@@ -16,121 +16,15 @@
// under the License.
#include "arrow/util/bpacking_avx512.h"
-#include "arrow/util/bpacking_avx512_generated.h"
-#include "arrow/util/logging.h"
+#include "arrow/util/bpacking_simd512_generated.h"
+#include "arrow/util/bpacking_simd_internal.h"
namespace arrow {
namespace internal {
int unpack32_avx512(const uint32_t* in, uint32_t* out, int batch_size, int num_bits) {
- batch_size = batch_size / 32 * 32;
- int num_loops = batch_size / 32;
-
- switch (num_bits) {
- case 0:
- for (int i = 0; i < num_loops; ++i) in = unpack0_32_avx512(in, out + i * 32);
- break;
- case 1:
- for (int i = 0; i < num_loops; ++i) in = unpack1_32_avx512(in, out + i * 32);
- break;
- case 2:
- for (int i = 0; i < num_loops; ++i) in = unpack2_32_avx512(in, out + i * 32);
- break;
- case 3:
- for (int i = 0; i < num_loops; ++i) in = unpack3_32_avx512(in, out + i * 32);
- break;
- case 4:
- for (int i = 0; i < num_loops; ++i) in = unpack4_32_avx512(in, out + i * 32);
- break;
- case 5:
- for (int i = 0; i < num_loops; ++i) in = unpack5_32_avx512(in, out + i * 32);
- break;
- case 6:
- for (int i = 0; i < num_loops; ++i) in = unpack6_32_avx512(in, out + i * 32);
- break;
- case 7:
- for (int i = 0; i < num_loops; ++i) in = unpack7_32_avx512(in, out + i * 32);
- break;
- case 8:
- for (int i = 0; i < num_loops; ++i) in = unpack8_32_avx512(in, out + i * 32);
- break;
- case 9:
- for (int i = 0; i < num_loops; ++i) in = unpack9_32_avx512(in, out + i * 32);
- break;
- case 10:
- for (int i = 0; i < num_loops; ++i) in = unpack10_32_avx512(in, out + i * 32);
- break;
- case 11:
- for (int i = 0; i < num_loops; ++i) in = unpack11_32_avx512(in, out + i * 32);
- break;
- case 12:
- for (int i = 0; i < num_loops; ++i) in = unpack12_32_avx512(in, out + i * 32);
- break;
- case 13:
- for (int i = 0; i < num_loops; ++i) in = unpack13_32_avx512(in, out + i * 32);
- break;
- case 14:
- for (int i = 0; i < num_loops; ++i) in = unpack14_32_avx512(in, out + i * 32);
- break;
- case 15:
- for (int i = 0; i < num_loops; ++i) in = unpack15_32_avx512(in, out + i * 32);
- break;
- case 16:
- for (int i = 0; i < num_loops; ++i) in = unpack16_32_avx512(in, out + i * 32);
- break;
- case 17:
- for (int i = 0; i < num_loops; ++i) in = unpack17_32_avx512(in, out + i * 32);
- break;
- case 18:
- for (int i = 0; i < num_loops; ++i) in = unpack18_32_avx512(in, out + i * 32);
- break;
- case 19:
- for (int i = 0; i < num_loops; ++i) in = unpack19_32_avx512(in, out + i * 32);
- break;
- case 20:
- for (int i = 0; i < num_loops; ++i) in = unpack20_32_avx512(in, out + i * 32);
- break;
- case 21:
- for (int i = 0; i < num_loops; ++i) in = unpack21_32_avx512(in, out + i * 32);
- break;
- case 22:
- for (int i = 0; i < num_loops; ++i) in = unpack22_32_avx512(in, out + i * 32);
- break;
- case 23:
- for (int i = 0; i < num_loops; ++i) in = unpack23_32_avx512(in, out + i * 32);
- break;
- case 24:
- for (int i = 0; i < num_loops; ++i) in = unpack24_32_avx512(in, out + i * 32);
- break;
- case 25:
- for (int i = 0; i < num_loops; ++i) in = unpack25_32_avx512(in, out + i * 32);
- break;
- case 26:
- for (int i = 0; i < num_loops; ++i) in = unpack26_32_avx512(in, out + i * 32);
- break;
- case 27:
- for (int i = 0; i < num_loops; ++i) in = unpack27_32_avx512(in, out + i * 32);
- break;
- case 28:
- for (int i = 0; i < num_loops; ++i) in = unpack28_32_avx512(in, out + i * 32);
- break;
- case 29:
- for (int i = 0; i < num_loops; ++i) in = unpack29_32_avx512(in, out + i * 32);
- break;
- case 30:
- for (int i = 0; i < num_loops; ++i) in = unpack30_32_avx512(in, out + i * 32);
- break;
- case 31:
- for (int i = 0; i < num_loops; ++i) in = unpack31_32_avx512(in, out + i * 32);
- break;
- case 32:
- for (int i = 0; i < num_loops; ++i) in = unpack32_32_avx512(in, out + i * 32);
- break;
- default:
- DCHECK(false) << "Unsupported num_bits";
- }
-
- return batch_size;
+ return unpack32_specialized<UnpackBits512<DispatchLevel::AVX512>>(in, out, batch_size,
+ num_bits);
}
} // namespace internal
diff --git a/cpp/src/arrow/util/bpacking_avx512_codegen.py b/cpp/src/arrow/util/bpacking_avx512_codegen.py
deleted file mode 100644
index df4d7d7..0000000
--- a/cpp/src/arrow/util/bpacking_avx512_codegen.py
+++ /dev/null
@@ -1,186 +0,0 @@
-#!/bin/python
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Usage: python bpacking_avx512_codegen.py > bpacking_avx512_generated.h
-
-
-def print_unpack_bit_func(bit):
- shift = 0
- shifts = []
- in_index = 0
- inls = []
- mask = (1 << bit) - 1
- bracket = "{"
-
- print(
- f"inline static const uint32_t* unpack{bit}_32_avx512(const uint32_t* in, uint32_t* out) {bracket}")
- print(" using ::arrow::util::SafeLoad;")
- print(" uint32_t mask = 0x%x;" % mask)
- print(" __m512i reg_shifts, reg_inls, reg_masks;")
- print(" __m512i results;")
-
- print("")
- for i in range(32):
- if shift + bit == 32:
- shifts.append(shift)
- inls.append(f"SafeLoad(in + {in_index})")
- in_index += 1
- shift = 0
- elif shift + bit > 32: # cross the boundary
- inls.append(
- f"SafeLoad(in + {in_index}) >> {shift} | SafeLoad(in + {in_index + 1}) << {32 - shift}")
- in_index += 1
- shift = bit - (32 - shift)
- shifts.append(0) # zero shift
- else:
- shifts.append(shift)
- inls.append(f"SafeLoad(in + {in_index})")
- shift += bit
-
- print(" reg_masks = _mm512_set1_epi32(mask);")
- print("")
- print(" // shift the first 16 outs")
- print(
- f" reg_shifts = _mm512_set_epi32({shifts[15]}, {shifts[14]}, {shifts[13]}, {shifts[12]},")
- print(
- f" {shifts[11]}, {shifts[10]}, {shifts[9]}, {shifts[8]},")
- print(
- f" {shifts[7]}, {shifts[6]}, {shifts[5]}, {shifts[4]},")
- print(
- f" {shifts[3]}, {shifts[2]}, {shifts[1]}, {shifts[0]});")
- print(f" reg_inls = _mm512_set_epi32({inls[15]}, {inls[14]},")
- print(f" {inls[13]}, {inls[12]},")
- print(f" {inls[11]}, {inls[10]},")
- print(f" {inls[9]}, {inls[8]},")
- print(f" {inls[7]}, {inls[6]},")
- print(f" {inls[5]}, {inls[4]},")
- print(f" {inls[3]}, {inls[2]},")
- print(f" {inls[1]}, {inls[0]});")
- print(
- " results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);")
- print(" _mm512_storeu_si512(out, results);")
- print(" out += 16;")
- print("")
- print(" // shift the second 16 outs")
- print(
- f" reg_shifts = _mm512_set_epi32({shifts[31]}, {shifts[30]}, {shifts[29]}, {shifts[28]},")
- print(
- f" {shifts[27]}, {shifts[26]}, {shifts[25]}, {shifts[24]},")
- print(
- f" {shifts[23]}, {shifts[22]}, {shifts[21]}, {shifts[20]},")
- print(
- f" {shifts[19]}, {shifts[18]}, {shifts[17]}, {shifts[16]});")
- print(f" reg_inls = _mm512_set_epi32({inls[31]}, {inls[30]},")
- print(f" {inls[29]}, {inls[28]},")
- print(f" {inls[27]}, {inls[26]},")
- print(f" {inls[25]}, {inls[24]},")
- print(f" {inls[23]}, {inls[22]},")
- print(f" {inls[21]}, {inls[20]},")
- print(f" {inls[19]}, {inls[18]},")
- print(f" {inls[17]}, {inls[16]});")
- print(
- " results = _mm512_and_epi32(_mm512_srlv_epi32(reg_inls, reg_shifts), reg_masks);")
- print(" _mm512_storeu_si512(out, results);")
- print(" out += 16;")
- print("")
- print(f" in += {bit};")
- print("")
- print(" return in;")
- print("}")
-
-
-def print_unpack_bit0_func():
- print(
- "inline static const uint32_t* unpack0_32_avx512(const uint32_t* in, uint32_t* out) {")
- print(" memset(out, 0x0, 32 * sizeof(*out));")
- print(" out += 32;")
- print("")
- print(" return in;")
- print("}")
-
-
-def print_unpack_bit32_func():
- print(
- "inline static const uint32_t* unpack32_32_avx512(const uint32_t* in, uint32_t* out) {")
- print(" memcpy(out, in, 32 * sizeof(*out));")
- print(" in += 32;")
- print(" out += 32;")
- print("")
- print(" return in;")
- print("}")
-
-
-def print_copyright():
- print(
- """// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.""")
-
-
-def print_note():
- print("//")
- print("// Automatically generated file; DO NOT EDIT.")
-
-
-def main():
- print_copyright()
- print_note()
- print("")
- print("#pragma once")
- print("")
- print("#include <stdint.h>")
- print("#include <string.h>")
- print("")
- print("#ifdef _MSC_VER")
- print("#include <intrin.h>")
- print("#else")
- print("#include <immintrin.h>")
- print("#endif")
- print("")
- print('#include "arrow/util/ubsan.h"')
- print("")
- print("namespace arrow {")
- print("namespace internal {")
- print("")
- print_unpack_bit0_func()
- print("")
- for i in range(1, 32):
- print_unpack_bit_func(i)
- print("")
- print_unpack_bit32_func()
- print("")
- print("} // namespace internal")
- print("} // namespace arrow")
-
-
-if __name__ == '__main__':
- main()
diff --git a/cpp/src/arrow/util/bpacking_neon.cc b/cpp/src/arrow/util/bpacking_neon.cc
new file mode 100644
index 0000000..a0bb5dc
--- /dev/null
+++ b/cpp/src/arrow/util/bpacking_neon.cc
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/util/bpacking_neon.h"
+#include "arrow/util/bpacking_simd128_generated.h"
+#include "arrow/util/bpacking_simd_internal.h"
+
+namespace arrow {
+namespace internal {
+
+int unpack32_neon(const uint32_t* in, uint32_t* out, int batch_size, int num_bits) {
+ return unpack32_specialized<UnpackBits128<DispatchLevel::NEON>>(in, out, batch_size,
+ num_bits);
+}
+
+} // namespace internal
+} // namespace arrow
diff --git a/cpp/src/arrow/util/bpacking_neon.h b/cpp/src/arrow/util/bpacking_neon.h
new file mode 100644
index 0000000..9d02cd5
--- /dev/null
+++ b/cpp/src/arrow/util/bpacking_neon.h
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <stdint.h>
+
+namespace arrow {
+namespace internal {
+
+int unpack32_neon(const uint32_t* in, uint32_t* out, int batch_size, int num_bits);
+
+} // namespace internal
+} // namespace arrow
diff --git a/cpp/src/arrow/util/bpacking_simd128_generated.h b/cpp/src/arrow/util/bpacking_simd128_generated.h
new file mode 100644
index 0000000..f7700fd
--- /dev/null
+++ b/cpp/src/arrow/util/bpacking_simd128_generated.h
@@ -0,0 +1,2138 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Automatically generated file; DO NOT EDIT.
+
+#pragma once
+
+#include <cstdint>
+#include <cstring>
+
+#include <xsimd/xsimd.hpp>
+
+#include "arrow/util/dispatch.h"
+#include "arrow/util/ubsan.h"
+
+namespace arrow {
+namespace internal {
+namespace {
+
+using ::arrow::util::SafeLoad;
+
+template <DispatchLevel level>
+struct UnpackBits128 {
+
+using simd_batch = xsimd::batch<uint32_t, 4>;
+
+inline static const uint32_t* unpack0_32(const uint32_t* in, uint32_t* out) {
+ memset(out, 0x0, 32 * sizeof(*out));
+ out += 32;
+
+ return in;
+}
+
+inline static const uint32_t* unpack1_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 1-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 0, 1, 2, 3 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 1-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 4, 5, 6, 7 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 1-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 8, 9, 10, 11 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 1-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 12, 13, 14, 15 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 1-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 16, 17, 18, 19 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 1-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 20, 21, 22, 23 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 1-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 24, 25, 26, 27 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 1-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 28, 29, 30, 31 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 1;
+ return in;
+}
+
+inline static const uint32_t* unpack2_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 2-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 0, 2, 4, 6 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 2-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 8, 10, 12, 14 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 2-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 16, 18, 20, 22 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 2-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 24, 26, 28, 30 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 2-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 2, 4, 6 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 2-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 8, 10, 12, 14 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 2-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 16, 18, 20, 22 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 2-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 24, 26, 28, 30 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 2;
+ return in;
+}
+
+inline static const uint32_t* unpack3_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 3-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 0, 3, 6, 9 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 3-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 12, 15, 18, 21 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 3-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 24, 27, 0, 1 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 3-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 4, 7, 10, 13 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 3-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 16, 19, 22, 25 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 3-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 31 | SafeLoad<uint32_t>(in + 2) << 1, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 28, 0, 2, 5 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 3-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 8, 11, 14, 17 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 3-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 20, 23, 26, 29 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 3;
+ return in;
+}
+
+inline static const uint32_t* unpack4_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xf;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 4-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 0, 4, 8, 12 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 4-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 16, 20, 24, 28 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 4-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 4, 8, 12 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 4-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 16, 20, 24, 28 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 4-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 0, 4, 8, 12 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 4-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 16, 20, 24, 28 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 4-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 0, 4, 8, 12 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 4-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 16, 20, 24, 28 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 4;
+ return in;
+}
+
+inline static const uint32_t* unpack5_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1f;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 5-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 0, 5, 10, 15 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 5-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 20, 25, 0, 3 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 5-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 8, 13, 18, 23 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 5-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 0, 1, 6, 11 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 5-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 31 | SafeLoad<uint32_t>(in + 3) << 1 };
+ shifts = simd_batch{ 16, 21, 26, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 5-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 4, 9, 14, 19 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 5-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 29 | SafeLoad<uint32_t>(in + 4) << 3, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 24, 0, 2, 7 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 5-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 12, 17, 22, 27 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 5;
+ return in;
+}
+
+inline static const uint32_t* unpack6_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3f;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 6-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 0, 6, 12, 18 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 6-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 24, 0, 4, 10 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 6-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 16, 22, 0, 2 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 6-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 8, 14, 20, 26 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 6-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 0, 6, 12, 18 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 6-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 24, 0, 4, 10 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 6-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 28 | SafeLoad<uint32_t>(in + 5) << 4, SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 16, 22, 0, 2 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 6-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 8, 14, 20, 26 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 6;
+ return in;
+}
+
+inline static const uint32_t* unpack7_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7f;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 7-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 0, 7, 14, 21 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 7-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0) >> 28 | SafeLoad<uint32_t>(in + 1) << 4, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 3, 10, 17 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 7-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 31 | SafeLoad<uint32_t>(in + 2) << 1, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 24, 0, 6, 13 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 7-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 27 | SafeLoad<uint32_t>(in + 3) << 5, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 20, 0, 2, 9 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 7-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2, SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 16, 23, 0, 5 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 7-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 26 | SafeLoad<uint32_t>(in + 5) << 6, SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 12, 19, 0, 1 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 7-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 29 | SafeLoad<uint32_t>(in + 6) << 3 };
+ shifts = simd_batch{ 8, 15, 22, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 7-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) };
+ shifts = simd_batch{ 4, 11, 18, 25 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 7;
+ return in;
+}
+
+inline static const uint32_t* unpack8_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 8-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 0, 8, 16, 24 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 8-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 8, 16, 24 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 8-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 0, 8, 16, 24 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 8-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 0, 8, 16, 24 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 8-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 0, 8, 16, 24 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 8-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 0, 8, 16, 24 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 8-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) };
+ shifts = simd_batch{ 0, 8, 16, 24 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 8-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) };
+ shifts = simd_batch{ 0, 8, 16, 24 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 8;
+ return in;
+}
+
+inline static const uint32_t* unpack9_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1ff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 9-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 27 | SafeLoad<uint32_t>(in + 1) << 5 };
+ shifts = simd_batch{ 0, 9, 18, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 9-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 31 | SafeLoad<uint32_t>(in + 2) << 1 };
+ shifts = simd_batch{ 4, 13, 22, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 9-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 26 | SafeLoad<uint32_t>(in + 3) << 6, SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 8, 17, 0, 3 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 9-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2, SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 12, 21, 0, 7 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 9-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 25 | SafeLoad<uint32_t>(in + 5) << 7, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 16, 0, 2, 11 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 9-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 29 | SafeLoad<uint32_t>(in + 6) << 3, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) };
+ shifts = simd_batch{ 20, 0, 6, 15 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 9-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6) >> 24 | SafeLoad<uint32_t>(in + 7) << 8, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) };
+ shifts = simd_batch{ 0, 1, 10, 19 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 9-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) };
+ shifts = simd_batch{ 0, 5, 14, 23 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 9;
+ return in;
+}
+
+inline static const uint32_t* unpack10_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3ff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 10-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2 };
+ shifts = simd_batch{ 0, 10, 20, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 10-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 8, 18, 0, 6 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 10-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 26 | SafeLoad<uint32_t>(in + 3) << 6, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 16, 0, 4, 14 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 10-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 0, 2, 12, 22 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 10-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 30 | SafeLoad<uint32_t>(in + 6) << 2 };
+ shifts = simd_batch{ 0, 10, 20, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 10-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 28 | SafeLoad<uint32_t>(in + 7) << 4, SafeLoad<uint32_t>(in + 7) };
+ shifts = simd_batch{ 8, 18, 0, 6 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 10-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 26 | SafeLoad<uint32_t>(in + 8) << 6, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) };
+ shifts = simd_batch{ 16, 0, 4, 14 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 10-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 8) >> 24 | SafeLoad<uint32_t>(in + 9) << 8, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) };
+ shifts = simd_batch{ 0, 2, 12, 22 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 10;
+ return in;
+}
+
+inline static const uint32_t* unpack11_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7ff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 11-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 22 | SafeLoad<uint32_t>(in + 1) << 10, SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 11, 0, 1 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 11-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 23 | SafeLoad<uint32_t>(in + 2) << 9, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 12, 0, 2, 13 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 11-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2) >> 24 | SafeLoad<uint32_t>(in + 3) << 8, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 25 | SafeLoad<uint32_t>(in + 4) << 7 };
+ shifts = simd_batch{ 0, 3, 14, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 11-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 26 | SafeLoad<uint32_t>(in + 5) << 6, SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 4, 15, 0, 5 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 11-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 27 | SafeLoad<uint32_t>(in + 6) << 5, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) };
+ shifts = simd_batch{ 16, 0, 6, 17 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 11-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6) >> 28 | SafeLoad<uint32_t>(in + 7) << 4, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 29 | SafeLoad<uint32_t>(in + 8) << 3 };
+ shifts = simd_batch{ 0, 7, 18, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 11-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 30 | SafeLoad<uint32_t>(in + 9) << 2, SafeLoad<uint32_t>(in + 9) };
+ shifts = simd_batch{ 8, 19, 0, 9 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 11-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 31 | SafeLoad<uint32_t>(in + 10) << 1, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) };
+ shifts = simd_batch{ 20, 0, 10, 21 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 11;
+ return in;
+}
+
+inline static const uint32_t* unpack12_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xfff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 12-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 24 | SafeLoad<uint32_t>(in + 1) << 8, SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 12, 0, 4 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 12-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 16, 0, 8, 20 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 12-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 0, 12, 0, 4 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 12-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 28 | SafeLoad<uint32_t>(in + 5) << 4, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 16, 0, 8, 20 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 12-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 24 | SafeLoad<uint32_t>(in + 7) << 8, SafeLoad<uint32_t>(in + 7) };
+ shifts = simd_batch{ 0, 12, 0, 4 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 12-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) };
+ shifts = simd_batch{ 16, 0, 8, 20 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 12-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 24 | SafeLoad<uint32_t>(in + 10) << 8, SafeLoad<uint32_t>(in + 10) };
+ shifts = simd_batch{ 0, 12, 0, 4 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 12-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 28 | SafeLoad<uint32_t>(in + 11) << 4, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) };
+ shifts = simd_batch{ 16, 0, 8, 20 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 12;
+ return in;
+}
+
+inline static const uint32_t* unpack13_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1fff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 13-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 26 | SafeLoad<uint32_t>(in + 1) << 6, SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 13, 0, 7 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 13-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1) >> 20 | SafeLoad<uint32_t>(in + 2) << 12, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 27 | SafeLoad<uint32_t>(in + 3) << 5 };
+ shifts = simd_batch{ 0, 1, 14, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 13-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 21 | SafeLoad<uint32_t>(in + 4) << 11, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 8, 0, 2, 15 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 13-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4) >> 28 | SafeLoad<uint32_t>(in + 5) << 4, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 22 | SafeLoad<uint32_t>(in + 6) << 10, SafeLoad<uint32_t>(in + 6) };
+ shifts = simd_batch{ 0, 9, 0, 3 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 13-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 29 | SafeLoad<uint32_t>(in + 7) << 3, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 23 | SafeLoad<uint32_t>(in + 8) << 9 };
+ shifts = simd_batch{ 16, 0, 10, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 13-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 30 | SafeLoad<uint32_t>(in + 9) << 2, SafeLoad<uint32_t>(in + 9) };
+ shifts = simd_batch{ 4, 17, 0, 11 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 13-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 9) >> 24 | SafeLoad<uint32_t>(in + 10) << 8, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 31 | SafeLoad<uint32_t>(in + 11) << 1 };
+ shifts = simd_batch{ 0, 5, 18, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 13-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 25 | SafeLoad<uint32_t>(in + 12) << 7, SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) };
+ shifts = simd_batch{ 12, 0, 6, 19 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 13;
+ return in;
+}
+
+inline static const uint32_t* unpack14_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3fff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 14-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 28 | SafeLoad<uint32_t>(in + 1) << 4, SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 14, 0, 10 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 14-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1) >> 24 | SafeLoad<uint32_t>(in + 2) << 8, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 20 | SafeLoad<uint32_t>(in + 3) << 12, SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 0, 6, 0, 2 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 14-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 26 | SafeLoad<uint32_t>(in + 5) << 6 };
+ shifts = simd_batch{ 16, 0, 12, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 14-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 22 | SafeLoad<uint32_t>(in + 6) << 10, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) };
+ shifts = simd_batch{ 8, 0, 4, 18 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 14-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8) };
+ shifts = simd_batch{ 0, 14, 0, 10 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 14-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 8) >> 24 | SafeLoad<uint32_t>(in + 9) << 8, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 20 | SafeLoad<uint32_t>(in + 10) << 12, SafeLoad<uint32_t>(in + 10) };
+ shifts = simd_batch{ 0, 6, 0, 2 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 14-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 30 | SafeLoad<uint32_t>(in + 11) << 2, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 26 | SafeLoad<uint32_t>(in + 12) << 6 };
+ shifts = simd_batch{ 16, 0, 12, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 14-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) >> 22 | SafeLoad<uint32_t>(in + 13) << 10, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) };
+ shifts = simd_batch{ 8, 0, 4, 18 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 14;
+ return in;
+}
+
+inline static const uint32_t* unpack15_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7fff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 15-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 15, 0, 13 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 15-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 26 | SafeLoad<uint32_t>(in + 3) << 6, SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 0, 11, 0, 9 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 15-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 22 | SafeLoad<uint32_t>(in + 5) << 10, SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 0, 7, 0, 5 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 15-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5) >> 20 | SafeLoad<uint32_t>(in + 6) << 12, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 18 | SafeLoad<uint32_t>(in + 7) << 14, SafeLoad<uint32_t>(in + 7) };
+ shifts = simd_batch{ 0, 3, 0, 1 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 15-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 31 | SafeLoad<uint32_t>(in + 8) << 1, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 29 | SafeLoad<uint32_t>(in + 9) << 3 };
+ shifts = simd_batch{ 16, 0, 14, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 15-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 27 | SafeLoad<uint32_t>(in + 10) << 5, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 25 | SafeLoad<uint32_t>(in + 11) << 7 };
+ shifts = simd_batch{ 12, 0, 10, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 15-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 23 | SafeLoad<uint32_t>(in + 12) << 9, SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) >> 21 | SafeLoad<uint32_t>(in + 13) << 11 };
+ shifts = simd_batch{ 8, 0, 6, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 15-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 19 | SafeLoad<uint32_t>(in + 14) << 13, SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) };
+ shifts = simd_batch{ 4, 0, 2, 17 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 15;
+ return in;
+}
+
+inline static const uint32_t* unpack16_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 16-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 16, 0, 16 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 16-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 0, 16, 0, 16 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 16-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 0, 16, 0, 16 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 16-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) };
+ shifts = simd_batch{ 0, 16, 0, 16 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 16-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) };
+ shifts = simd_batch{ 0, 16, 0, 16 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 16-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) };
+ shifts = simd_batch{ 0, 16, 0, 16 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 16-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) };
+ shifts = simd_batch{ 0, 16, 0, 16 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 16-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) };
+ shifts = simd_batch{ 0, 16, 0, 16 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 16;
+ return in;
+}
+
+inline static const uint32_t* unpack17_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1ffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 17-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 17 | SafeLoad<uint32_t>(in + 1) << 15, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 19 | SafeLoad<uint32_t>(in + 2) << 13 };
+ shifts = simd_batch{ 0, 0, 2, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 17-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 21 | SafeLoad<uint32_t>(in + 3) << 11, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 23 | SafeLoad<uint32_t>(in + 4) << 9 };
+ shifts = simd_batch{ 4, 0, 6, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 17-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 25 | SafeLoad<uint32_t>(in + 5) << 7, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 27 | SafeLoad<uint32_t>(in + 6) << 5 };
+ shifts = simd_batch{ 8, 0, 10, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 17-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 29 | SafeLoad<uint32_t>(in + 7) << 3, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 31 | SafeLoad<uint32_t>(in + 8) << 1 };
+ shifts = simd_batch{ 12, 0, 14, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 17-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 8) >> 16 | SafeLoad<uint32_t>(in + 9) << 16, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 18 | SafeLoad<uint32_t>(in + 10) << 14, SafeLoad<uint32_t>(in + 10) };
+ shifts = simd_batch{ 0, 1, 0, 3 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 17-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 10) >> 20 | SafeLoad<uint32_t>(in + 11) << 12, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 22 | SafeLoad<uint32_t>(in + 12) << 10, SafeLoad<uint32_t>(in + 12) };
+ shifts = simd_batch{ 0, 5, 0, 7 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 17-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 12) >> 24 | SafeLoad<uint32_t>(in + 13) << 8, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 26 | SafeLoad<uint32_t>(in + 14) << 6, SafeLoad<uint32_t>(in + 14) };
+ shifts = simd_batch{ 0, 9, 0, 11 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 17-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 14) >> 28 | SafeLoad<uint32_t>(in + 15) << 4, SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 30 | SafeLoad<uint32_t>(in + 16) << 2, SafeLoad<uint32_t>(in + 16) };
+ shifts = simd_batch{ 0, 13, 0, 15 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 17;
+ return in;
+}
+
+inline static const uint32_t* unpack18_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3ffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 18-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 18 | SafeLoad<uint32_t>(in + 1) << 14, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 22 | SafeLoad<uint32_t>(in + 2) << 10 };
+ shifts = simd_batch{ 0, 0, 4, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 18-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 26 | SafeLoad<uint32_t>(in + 3) << 6, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2 };
+ shifts = simd_batch{ 8, 0, 12, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 18-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4) >> 16 | SafeLoad<uint32_t>(in + 5) << 16, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 20 | SafeLoad<uint32_t>(in + 6) << 12, SafeLoad<uint32_t>(in + 6) };
+ shifts = simd_batch{ 0, 2, 0, 6 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 18-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6) >> 24 | SafeLoad<uint32_t>(in + 7) << 8, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8) };
+ shifts = simd_batch{ 0, 10, 0, 14 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 18-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 18 | SafeLoad<uint32_t>(in + 10) << 14, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 22 | SafeLoad<uint32_t>(in + 11) << 10 };
+ shifts = simd_batch{ 0, 0, 4, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 18-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 26 | SafeLoad<uint32_t>(in + 12) << 6, SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) >> 30 | SafeLoad<uint32_t>(in + 13) << 2 };
+ shifts = simd_batch{ 8, 0, 12, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 18-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 13) >> 16 | SafeLoad<uint32_t>(in + 14) << 16, SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) >> 20 | SafeLoad<uint32_t>(in + 15) << 12, SafeLoad<uint32_t>(in + 15) };
+ shifts = simd_batch{ 0, 2, 0, 6 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 18-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 15) >> 24 | SafeLoad<uint32_t>(in + 16) << 8, SafeLoad<uint32_t>(in + 16), SafeLoad<uint32_t>(in + 16) >> 28 | SafeLoad<uint32_t>(in + 17) << 4, SafeLoad<uint32_t>(in + 17) };
+ shifts = simd_batch{ 0, 10, 0, 14 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 18;
+ return in;
+}
+
+inline static const uint32_t* unpack19_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7ffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 19-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 19 | SafeLoad<uint32_t>(in + 1) << 13, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 25 | SafeLoad<uint32_t>(in + 2) << 7 };
+ shifts = simd_batch{ 0, 0, 6, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 19-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 31 | SafeLoad<uint32_t>(in + 3) << 1, SafeLoad<uint32_t>(in + 3) >> 18 | SafeLoad<uint32_t>(in + 4) << 14, SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 12, 0, 0, 5 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 19-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4) >> 24 | SafeLoad<uint32_t>(in + 5) << 8, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 30 | SafeLoad<uint32_t>(in + 6) << 2, SafeLoad<uint32_t>(in + 6) >> 17 | SafeLoad<uint32_t>(in + 7) << 15 };
+ shifts = simd_batch{ 0, 11, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 19-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 23 | SafeLoad<uint32_t>(in + 8) << 9, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 29 | SafeLoad<uint32_t>(in + 9) << 3 };
+ shifts = simd_batch{ 4, 0, 10, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 19-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 9) >> 16 | SafeLoad<uint32_t>(in + 10) << 16, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 22 | SafeLoad<uint32_t>(in + 11) << 10, SafeLoad<uint32_t>(in + 11) };
+ shifts = simd_batch{ 0, 3, 0, 9 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 19-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 11) >> 28 | SafeLoad<uint32_t>(in + 12) << 4, SafeLoad<uint32_t>(in + 12) >> 15 | SafeLoad<uint32_t>(in + 13) << 17, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 21 | SafeLoad<uint32_t>(in + 14) << 11 };
+ shifts = simd_batch{ 0, 0, 2, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 19-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) >> 27 | SafeLoad<uint32_t>(in + 15) << 5, SafeLoad<uint32_t>(in + 15) >> 14 | SafeLoad<uint32_t>(in + 16) << 18, SafeLoad<uint32_t>(in + 16) };
+ shifts = simd_batch{ 8, 0, 0, 1 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 19-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 16) >> 20 | SafeLoad<uint32_t>(in + 17) << 12, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 26 | SafeLoad<uint32_t>(in + 18) << 6, SafeLoad<uint32_t>(in + 18) };
+ shifts = simd_batch{ 0, 7, 0, 13 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 19;
+ return in;
+}
+
+inline static const uint32_t* unpack20_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xfffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 20-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 20 | SafeLoad<uint32_t>(in + 1) << 12, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4 };
+ shifts = simd_batch{ 0, 0, 8, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 20-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2) >> 16 | SafeLoad<uint32_t>(in + 3) << 16, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 0, 4, 0, 12 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 20-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 20 | SafeLoad<uint32_t>(in + 6) << 12, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 28 | SafeLoad<uint32_t>(in + 7) << 4 };
+ shifts = simd_batch{ 0, 0, 8, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 20-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7) >> 16 | SafeLoad<uint32_t>(in + 8) << 16, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 24 | SafeLoad<uint32_t>(in + 9) << 8, SafeLoad<uint32_t>(in + 9) };
+ shifts = simd_batch{ 0, 4, 0, 12 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 20-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 20 | SafeLoad<uint32_t>(in + 11) << 12, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 28 | SafeLoad<uint32_t>(in + 12) << 4 };
+ shifts = simd_batch{ 0, 0, 8, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 20-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 12) >> 16 | SafeLoad<uint32_t>(in + 13) << 16, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 24 | SafeLoad<uint32_t>(in + 14) << 8, SafeLoad<uint32_t>(in + 14) };
+ shifts = simd_batch{ 0, 4, 0, 12 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 20-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 20 | SafeLoad<uint32_t>(in + 16) << 12, SafeLoad<uint32_t>(in + 16), SafeLoad<uint32_t>(in + 16) >> 28 | SafeLoad<uint32_t>(in + 17) << 4 };
+ shifts = simd_batch{ 0, 0, 8, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 20-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 17) >> 16 | SafeLoad<uint32_t>(in + 18) << 16, SafeLoad<uint32_t>(in + 18), SafeLoad<uint32_t>(in + 18) >> 24 | SafeLoad<uint32_t>(in + 19) << 8, SafeLoad<uint32_t>(in + 19) };
+ shifts = simd_batch{ 0, 4, 0, 12 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 20;
+ return in;
+}
+
+inline static const uint32_t* unpack21_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1fffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 21-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 21 | SafeLoad<uint32_t>(in + 1) << 11, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 31 | SafeLoad<uint32_t>(in + 2) << 1 };
+ shifts = simd_batch{ 0, 0, 10, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 21-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2) >> 20 | SafeLoad<uint32_t>(in + 3) << 12, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2, SafeLoad<uint32_t>(in + 4) >> 19 | SafeLoad<uint32_t>(in + 5) << 13 };
+ shifts = simd_batch{ 0, 9, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 21-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 29 | SafeLoad<uint32_t>(in + 6) << 3, SafeLoad<uint32_t>(in + 6) >> 18 | SafeLoad<uint32_t>(in + 7) << 14, SafeLoad<uint32_t>(in + 7) };
+ shifts = simd_batch{ 8, 0, 0, 7 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 21-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8) >> 17 | SafeLoad<uint32_t>(in + 9) << 15, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 27 | SafeLoad<uint32_t>(in + 10) << 5 };
+ shifts = simd_batch{ 0, 0, 6, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 21-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 10) >> 16 | SafeLoad<uint32_t>(in + 11) << 16, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 26 | SafeLoad<uint32_t>(in + 12) << 6, SafeLoad<uint32_t>(in + 12) >> 15 | SafeLoad<uint32_t>(in + 13) << 17 };
+ shifts = simd_batch{ 0, 5, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 21-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 25 | SafeLoad<uint32_t>(in + 14) << 7, SafeLoad<uint32_t>(in + 14) >> 14 | SafeLoad<uint32_t>(in + 15) << 18, SafeLoad<uint32_t>(in + 15) };
+ shifts = simd_batch{ 4, 0, 0, 3 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 21-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 15) >> 24 | SafeLoad<uint32_t>(in + 16) << 8, SafeLoad<uint32_t>(in + 16) >> 13 | SafeLoad<uint32_t>(in + 17) << 19, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 23 | SafeLoad<uint32_t>(in + 18) << 9 };
+ shifts = simd_batch{ 0, 0, 2, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 21-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 18) >> 12 | SafeLoad<uint32_t>(in + 19) << 20, SafeLoad<uint32_t>(in + 19), SafeLoad<uint32_t>(in + 19) >> 22 | SafeLoad<uint32_t>(in + 20) << 10, SafeLoad<uint32_t>(in + 20) };
+ shifts = simd_batch{ 0, 1, 0, 11 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 21;
+ return in;
+}
+
+inline static const uint32_t* unpack22_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3fffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 22-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 22 | SafeLoad<uint32_t>(in + 1) << 10, SafeLoad<uint32_t>(in + 1) >> 12 | SafeLoad<uint32_t>(in + 2) << 20, SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 0, 0, 0, 2 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 22-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2) >> 24 | SafeLoad<uint32_t>(in + 3) << 8, SafeLoad<uint32_t>(in + 3) >> 14 | SafeLoad<uint32_t>(in + 4) << 18, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 26 | SafeLoad<uint32_t>(in + 5) << 6 };
+ shifts = simd_batch{ 0, 0, 4, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 22-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5) >> 16 | SafeLoad<uint32_t>(in + 6) << 16, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 28 | SafeLoad<uint32_t>(in + 7) << 4, SafeLoad<uint32_t>(in + 7) >> 18 | SafeLoad<uint32_t>(in + 8) << 14 };
+ shifts = simd_batch{ 0, 6, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 22-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 30 | SafeLoad<uint32_t>(in + 9) << 2, SafeLoad<uint32_t>(in + 9) >> 20 | SafeLoad<uint32_t>(in + 10) << 12, SafeLoad<uint32_t>(in + 10) };
+ shifts = simd_batch{ 8, 0, 0, 10 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 22-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 22 | SafeLoad<uint32_t>(in + 12) << 10, SafeLoad<uint32_t>(in + 12) >> 12 | SafeLoad<uint32_t>(in + 13) << 20, SafeLoad<uint32_t>(in + 13) };
+ shifts = simd_batch{ 0, 0, 0, 2 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 22-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 13) >> 24 | SafeLoad<uint32_t>(in + 14) << 8, SafeLoad<uint32_t>(in + 14) >> 14 | SafeLoad<uint32_t>(in + 15) << 18, SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 26 | SafeLoad<uint32_t>(in + 16) << 6 };
+ shifts = simd_batch{ 0, 0, 4, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 22-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 16) >> 16 | SafeLoad<uint32_t>(in + 17) << 16, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 28 | SafeLoad<uint32_t>(in + 18) << 4, SafeLoad<uint32_t>(in + 18) >> 18 | SafeLoad<uint32_t>(in + 19) << 14 };
+ shifts = simd_batch{ 0, 6, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 22-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 19), SafeLoad<uint32_t>(in + 19) >> 30 | SafeLoad<uint32_t>(in + 20) << 2, SafeLoad<uint32_t>(in + 20) >> 20 | SafeLoad<uint32_t>(in + 21) << 12, SafeLoad<uint32_t>(in + 21) };
+ shifts = simd_batch{ 8, 0, 0, 10 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 22;
+ return in;
+}
+
+inline static const uint32_t* unpack23_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7fffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 23-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 23 | SafeLoad<uint32_t>(in + 1) << 9, SafeLoad<uint32_t>(in + 1) >> 14 | SafeLoad<uint32_t>(in + 2) << 18, SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 0, 0, 0, 5 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 23-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2) >> 28 | SafeLoad<uint32_t>(in + 3) << 4, SafeLoad<uint32_t>(in + 3) >> 19 | SafeLoad<uint32_t>(in + 4) << 13, SafeLoad<uint32_t>(in + 4) >> 10 | SafeLoad<uint32_t>(in + 5) << 22, SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 0, 0, 0, 1 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 23-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5) >> 24 | SafeLoad<uint32_t>(in + 6) << 8, SafeLoad<uint32_t>(in + 6) >> 15 | SafeLoad<uint32_t>(in + 7) << 17, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 29 | SafeLoad<uint32_t>(in + 8) << 3 };
+ shifts = simd_batch{ 0, 0, 6, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 23-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 8) >> 20 | SafeLoad<uint32_t>(in + 9) << 12, SafeLoad<uint32_t>(in + 9) >> 11 | SafeLoad<uint32_t>(in + 10) << 21, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 25 | SafeLoad<uint32_t>(in + 11) << 7 };
+ shifts = simd_batch{ 0, 0, 2, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 23-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 11) >> 16 | SafeLoad<uint32_t>(in + 12) << 16, SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) >> 30 | SafeLoad<uint32_t>(in + 13) << 2, SafeLoad<uint32_t>(in + 13) >> 21 | SafeLoad<uint32_t>(in + 14) << 11 };
+ shifts = simd_batch{ 0, 7, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 23-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 14) >> 12 | SafeLoad<uint32_t>(in + 15) << 20, SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 26 | SafeLoad<uint32_t>(in + 16) << 6, SafeLoad<uint32_t>(in + 16) >> 17 | SafeLoad<uint32_t>(in + 17) << 15 };
+ shifts = simd_batch{ 0, 3, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 23-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 31 | SafeLoad<uint32_t>(in + 18) << 1, SafeLoad<uint32_t>(in + 18) >> 22 | SafeLoad<uint32_t>(in + 19) << 10, SafeLoad<uint32_t>(in + 19) >> 13 | SafeLoad<uint32_t>(in + 20) << 19 };
+ shifts = simd_batch{ 8, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 23-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 20), SafeLoad<uint32_t>(in + 20) >> 27 | SafeLoad<uint32_t>(in + 21) << 5, SafeLoad<uint32_t>(in + 21) >> 18 | SafeLoad<uint32_t>(in + 22) << 14, SafeLoad<uint32_t>(in + 22) };
+ shifts = simd_batch{ 4, 0, 0, 9 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 23;
+ return in;
+}
+
+inline static const uint32_t* unpack24_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 24-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 24 | SafeLoad<uint32_t>(in + 1) << 8, SafeLoad<uint32_t>(in + 1) >> 16 | SafeLoad<uint32_t>(in + 2) << 16, SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 0, 0, 0, 8 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 24-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4) >> 16 | SafeLoad<uint32_t>(in + 5) << 16, SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 0, 0, 0, 8 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 24-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 24 | SafeLoad<uint32_t>(in + 7) << 8, SafeLoad<uint32_t>(in + 7) >> 16 | SafeLoad<uint32_t>(in + 8) << 16, SafeLoad<uint32_t>(in + 8) };
+ shifts = simd_batch{ 0, 0, 0, 8 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 24-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 24 | SafeLoad<uint32_t>(in + 10) << 8, SafeLoad<uint32_t>(in + 10) >> 16 | SafeLoad<uint32_t>(in + 11) << 16, SafeLoad<uint32_t>(in + 11) };
+ shifts = simd_batch{ 0, 0, 0, 8 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 24-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) >> 24 | SafeLoad<uint32_t>(in + 13) << 8, SafeLoad<uint32_t>(in + 13) >> 16 | SafeLoad<uint32_t>(in + 14) << 16, SafeLoad<uint32_t>(in + 14) };
+ shifts = simd_batch{ 0, 0, 0, 8 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 24-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 24 | SafeLoad<uint32_t>(in + 16) << 8, SafeLoad<uint32_t>(in + 16) >> 16 | SafeLoad<uint32_t>(in + 17) << 16, SafeLoad<uint32_t>(in + 17) };
+ shifts = simd_batch{ 0, 0, 0, 8 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 24-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 18), SafeLoad<uint32_t>(in + 18) >> 24 | SafeLoad<uint32_t>(in + 19) << 8, SafeLoad<uint32_t>(in + 19) >> 16 | SafeLoad<uint32_t>(in + 20) << 16, SafeLoad<uint32_t>(in + 20) };
+ shifts = simd_batch{ 0, 0, 0, 8 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 24-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 21), SafeLoad<uint32_t>(in + 21) >> 24 | SafeLoad<uint32_t>(in + 22) << 8, SafeLoad<uint32_t>(in + 22) >> 16 | SafeLoad<uint32_t>(in + 23) << 16, SafeLoad<uint32_t>(in + 23) };
+ shifts = simd_batch{ 0, 0, 0, 8 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 24;
+ return in;
+}
+
+inline static const uint32_t* unpack25_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1ffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 25-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 25 | SafeLoad<uint32_t>(in + 1) << 7, SafeLoad<uint32_t>(in + 1) >> 18 | SafeLoad<uint32_t>(in + 2) << 14, SafeLoad<uint32_t>(in + 2) >> 11 | SafeLoad<uint32_t>(in + 3) << 21 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 25-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 29 | SafeLoad<uint32_t>(in + 4) << 3, SafeLoad<uint32_t>(in + 4) >> 22 | SafeLoad<uint32_t>(in + 5) << 10, SafeLoad<uint32_t>(in + 5) >> 15 | SafeLoad<uint32_t>(in + 6) << 17 };
+ shifts = simd_batch{ 4, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 25-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6) >> 8 | SafeLoad<uint32_t>(in + 7) << 24, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 26 | SafeLoad<uint32_t>(in + 8) << 6, SafeLoad<uint32_t>(in + 8) >> 19 | SafeLoad<uint32_t>(in + 9) << 13 };
+ shifts = simd_batch{ 0, 1, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 25-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 9) >> 12 | SafeLoad<uint32_t>(in + 10) << 20, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 30 | SafeLoad<uint32_t>(in + 11) << 2, SafeLoad<uint32_t>(in + 11) >> 23 | SafeLoad<uint32_t>(in + 12) << 9 };
+ shifts = simd_batch{ 0, 5, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 25-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 12) >> 16 | SafeLoad<uint32_t>(in + 13) << 16, SafeLoad<uint32_t>(in + 13) >> 9 | SafeLoad<uint32_t>(in + 14) << 23, SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) >> 27 | SafeLoad<uint32_t>(in + 15) << 5 };
+ shifts = simd_batch{ 0, 0, 2, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 25-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 15) >> 20 | SafeLoad<uint32_t>(in + 16) << 12, SafeLoad<uint32_t>(in + 16) >> 13 | SafeLoad<uint32_t>(in + 17) << 19, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 31 | SafeLoad<uint32_t>(in + 18) << 1 };
+ shifts = simd_batch{ 0, 0, 6, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 25-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 18) >> 24 | SafeLoad<uint32_t>(in + 19) << 8, SafeLoad<uint32_t>(in + 19) >> 17 | SafeLoad<uint32_t>(in + 20) << 15, SafeLoad<uint32_t>(in + 20) >> 10 | SafeLoad<uint32_t>(in + 21) << 22, SafeLoad<uint32_t>(in + 21) };
+ shifts = simd_batch{ 0, 0, 0, 3 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 25-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 21) >> 28 | SafeLoad<uint32_t>(in + 22) << 4, SafeLoad<uint32_t>(in + 22) >> 21 | SafeLoad<uint32_t>(in + 23) << 11, SafeLoad<uint32_t>(in + 23) >> 14 | SafeLoad<uint32_t>(in + 24) << 18, SafeLoad<uint32_t>(in + 24) };
+ shifts = simd_batch{ 0, 0, 0, 7 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 25;
+ return in;
+}
+
+inline static const uint32_t* unpack26_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3ffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 26-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 26 | SafeLoad<uint32_t>(in + 1) << 6, SafeLoad<uint32_t>(in + 1) >> 20 | SafeLoad<uint32_t>(in + 2) << 12, SafeLoad<uint32_t>(in + 2) >> 14 | SafeLoad<uint32_t>(in + 3) << 18 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 26-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3) >> 8 | SafeLoad<uint32_t>(in + 4) << 24, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 28 | SafeLoad<uint32_t>(in + 5) << 4, SafeLoad<uint32_t>(in + 5) >> 22 | SafeLoad<uint32_t>(in + 6) << 10 };
+ shifts = simd_batch{ 0, 2, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 26-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6) >> 16 | SafeLoad<uint32_t>(in + 7) << 16, SafeLoad<uint32_t>(in + 7) >> 10 | SafeLoad<uint32_t>(in + 8) << 22, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 30 | SafeLoad<uint32_t>(in + 9) << 2 };
+ shifts = simd_batch{ 0, 0, 4, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 26-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 9) >> 24 | SafeLoad<uint32_t>(in + 10) << 8, SafeLoad<uint32_t>(in + 10) >> 18 | SafeLoad<uint32_t>(in + 11) << 14, SafeLoad<uint32_t>(in + 11) >> 12 | SafeLoad<uint32_t>(in + 12) << 20, SafeLoad<uint32_t>(in + 12) };
+ shifts = simd_batch{ 0, 0, 0, 6 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 26-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 26 | SafeLoad<uint32_t>(in + 14) << 6, SafeLoad<uint32_t>(in + 14) >> 20 | SafeLoad<uint32_t>(in + 15) << 12, SafeLoad<uint32_t>(in + 15) >> 14 | SafeLoad<uint32_t>(in + 16) << 18 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 26-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 16) >> 8 | SafeLoad<uint32_t>(in + 17) << 24, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 28 | SafeLoad<uint32_t>(in + 18) << 4, SafeLoad<uint32_t>(in + 18) >> 22 | SafeLoad<uint32_t>(in + 19) << 10 };
+ shifts = simd_batch{ 0, 2, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 26-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 19) >> 16 | SafeLoad<uint32_t>(in + 20) << 16, SafeLoad<uint32_t>(in + 20) >> 10 | SafeLoad<uint32_t>(in + 21) << 22, SafeLoad<uint32_t>(in + 21), SafeLoad<uint32_t>(in + 21) >> 30 | SafeLoad<uint32_t>(in + 22) << 2 };
+ shifts = simd_batch{ 0, 0, 4, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 26-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 22) >> 24 | SafeLoad<uint32_t>(in + 23) << 8, SafeLoad<uint32_t>(in + 23) >> 18 | SafeLoad<uint32_t>(in + 24) << 14, SafeLoad<uint32_t>(in + 24) >> 12 | SafeLoad<uint32_t>(in + 25) << 20, SafeLoad<uint32_t>(in + 25) };
+ shifts = simd_batch{ 0, 0, 0, 6 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 26;
+ return in;
+}
+
+inline static const uint32_t* unpack27_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7ffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 27-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 27 | SafeLoad<uint32_t>(in + 1) << 5, SafeLoad<uint32_t>(in + 1) >> 22 | SafeLoad<uint32_t>(in + 2) << 10, SafeLoad<uint32_t>(in + 2) >> 17 | SafeLoad<uint32_t>(in + 3) << 15 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 27-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3) >> 12 | SafeLoad<uint32_t>(in + 4) << 20, SafeLoad<uint32_t>(in + 4) >> 7 | SafeLoad<uint32_t>(in + 5) << 25, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 29 | SafeLoad<uint32_t>(in + 6) << 3 };
+ shifts = simd_batch{ 0, 0, 2, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 27-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6) >> 24 | SafeLoad<uint32_t>(in + 7) << 8, SafeLoad<uint32_t>(in + 7) >> 19 | SafeLoad<uint32_t>(in + 8) << 13, SafeLoad<uint32_t>(in + 8) >> 14 | SafeLoad<uint32_t>(in + 9) << 18, SafeLoad<uint32_t>(in + 9) >> 9 | SafeLoad<uint32_t>(in + 10) << 23 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 27-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 31 | SafeLoad<uint32_t>(in + 11) << 1, SafeLoad<uint32_t>(in + 11) >> 26 | SafeLoad<uint32_t>(in + 12) << 6, SafeLoad<uint32_t>(in + 12) >> 21 | SafeLoad<uint32_t>(in + 13) << 11 };
+ shifts = simd_batch{ 4, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 27-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 13) >> 16 | SafeLoad<uint32_t>(in + 14) << 16, SafeLoad<uint32_t>(in + 14) >> 11 | SafeLoad<uint32_t>(in + 15) << 21, SafeLoad<uint32_t>(in + 15) >> 6 | SafeLoad<uint32_t>(in + 16) << 26, SafeLoad<uint32_t>(in + 16) };
+ shifts = simd_batch{ 0, 0, 0, 1 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 27-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 16) >> 28 | SafeLoad<uint32_t>(in + 17) << 4, SafeLoad<uint32_t>(in + 17) >> 23 | SafeLoad<uint32_t>(in + 18) << 9, SafeLoad<uint32_t>(in + 18) >> 18 | SafeLoad<uint32_t>(in + 19) << 14, SafeLoad<uint32_t>(in + 19) >> 13 | SafeLoad<uint32_t>(in + 20) << 19 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 27-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 20) >> 8 | SafeLoad<uint32_t>(in + 21) << 24, SafeLoad<uint32_t>(in + 21), SafeLoad<uint32_t>(in + 21) >> 30 | SafeLoad<uint32_t>(in + 22) << 2, SafeLoad<uint32_t>(in + 22) >> 25 | SafeLoad<uint32_t>(in + 23) << 7 };
+ shifts = simd_batch{ 0, 3, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 27-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 23) >> 20 | SafeLoad<uint32_t>(in + 24) << 12, SafeLoad<uint32_t>(in + 24) >> 15 | SafeLoad<uint32_t>(in + 25) << 17, SafeLoad<uint32_t>(in + 25) >> 10 | SafeLoad<uint32_t>(in + 26) << 22, SafeLoad<uint32_t>(in + 26) };
+ shifts = simd_batch{ 0, 0, 0, 5 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 27;
+ return in;
+}
+
+inline static const uint32_t* unpack28_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xfffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 28-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 28 | SafeLoad<uint32_t>(in + 1) << 4, SafeLoad<uint32_t>(in + 1) >> 24 | SafeLoad<uint32_t>(in + 2) << 8, SafeLoad<uint32_t>(in + 2) >> 20 | SafeLoad<uint32_t>(in + 3) << 12 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 28-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3) >> 16 | SafeLoad<uint32_t>(in + 4) << 16, SafeLoad<uint32_t>(in + 4) >> 12 | SafeLoad<uint32_t>(in + 5) << 20, SafeLoad<uint32_t>(in + 5) >> 8 | SafeLoad<uint32_t>(in + 6) << 24, SafeLoad<uint32_t>(in + 6) };
+ shifts = simd_batch{ 0, 0, 0, 4 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 28-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8) >> 24 | SafeLoad<uint32_t>(in + 9) << 8, SafeLoad<uint32_t>(in + 9) >> 20 | SafeLoad<uint32_t>(in + 10) << 12 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 28-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 10) >> 16 | SafeLoad<uint32_t>(in + 11) << 16, SafeLoad<uint32_t>(in + 11) >> 12 | SafeLoad<uint32_t>(in + 12) << 20, SafeLoad<uint32_t>(in + 12) >> 8 | SafeLoad<uint32_t>(in + 13) << 24, SafeLoad<uint32_t>(in + 13) };
+ shifts = simd_batch{ 0, 0, 0, 4 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 28-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) >> 28 | SafeLoad<uint32_t>(in + 15) << 4, SafeLoad<uint32_t>(in + 15) >> 24 | SafeLoad<uint32_t>(in + 16) << 8, SafeLoad<uint32_t>(in + 16) >> 20 | SafeLoad<uint32_t>(in + 17) << 12 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 28-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 17) >> 16 | SafeLoad<uint32_t>(in + 18) << 16, SafeLoad<uint32_t>(in + 18) >> 12 | SafeLoad<uint32_t>(in + 19) << 20, SafeLoad<uint32_t>(in + 19) >> 8 | SafeLoad<uint32_t>(in + 20) << 24, SafeLoad<uint32_t>(in + 20) };
+ shifts = simd_batch{ 0, 0, 0, 4 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 28-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 21), SafeLoad<uint32_t>(in + 21) >> 28 | SafeLoad<uint32_t>(in + 22) << 4, SafeLoad<uint32_t>(in + 22) >> 24 | SafeLoad<uint32_t>(in + 23) << 8, SafeLoad<uint32_t>(in + 23) >> 20 | SafeLoad<uint32_t>(in + 24) << 12 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 28-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 24) >> 16 | SafeLoad<uint32_t>(in + 25) << 16, SafeLoad<uint32_t>(in + 25) >> 12 | SafeLoad<uint32_t>(in + 26) << 20, SafeLoad<uint32_t>(in + 26) >> 8 | SafeLoad<uint32_t>(in + 27) << 24, SafeLoad<uint32_t>(in + 27) };
+ shifts = simd_batch{ 0, 0, 0, 4 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 28;
+ return in;
+}
+
+inline static const uint32_t* unpack29_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1fffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 29-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 29 | SafeLoad<uint32_t>(in + 1) << 3, SafeLoad<uint32_t>(in + 1) >> 26 | SafeLoad<uint32_t>(in + 2) << 6, SafeLoad<uint32_t>(in + 2) >> 23 | SafeLoad<uint32_t>(in + 3) << 9 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 29-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3) >> 20 | SafeLoad<uint32_t>(in + 4) << 12, SafeLoad<uint32_t>(in + 4) >> 17 | SafeLoad<uint32_t>(in + 5) << 15, SafeLoad<uint32_t>(in + 5) >> 14 | SafeLoad<uint32_t>(in + 6) << 18, SafeLoad<uint32_t>(in + 6) >> 11 | SafeLoad<uint32_t>(in + 7) << 21 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 29-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7) >> 8 | SafeLoad<uint32_t>(in + 8) << 24, SafeLoad<uint32_t>(in + 8) >> 5 | SafeLoad<uint32_t>(in + 9) << 27, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 31 | SafeLoad<uint32_t>(in + 10) << 1 };
+ shifts = simd_batch{ 0, 0, 2, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 29-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 10) >> 28 | SafeLoad<uint32_t>(in + 11) << 4, SafeLoad<uint32_t>(in + 11) >> 25 | SafeLoad<uint32_t>(in + 12) << 7, SafeLoad<uint32_t>(in + 12) >> 22 | SafeLoad<uint32_t>(in + 13) << 10, SafeLoad<uint32_t>(in + 13) >> 19 | SafeLoad<uint32_t>(in + 14) << 13 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 29-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 14) >> 16 | SafeLoad<uint32_t>(in + 15) << 16, SafeLoad<uint32_t>(in + 15) >> 13 | SafeLoad<uint32_t>(in + 16) << 19, SafeLoad<uint32_t>(in + 16) >> 10 | SafeLoad<uint32_t>(in + 17) << 22, SafeLoad<uint32_t>(in + 17) >> 7 | SafeLoad<uint32_t>(in + 18) << 25 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 29-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 18) >> 4 | SafeLoad<uint32_t>(in + 19) << 28, SafeLoad<uint32_t>(in + 19), SafeLoad<uint32_t>(in + 19) >> 30 | SafeLoad<uint32_t>(in + 20) << 2, SafeLoad<uint32_t>(in + 20) >> 27 | SafeLoad<uint32_t>(in + 21) << 5 };
+ shifts = simd_batch{ 0, 1, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 29-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 21) >> 24 | SafeLoad<uint32_t>(in + 22) << 8, SafeLoad<uint32_t>(in + 22) >> 21 | SafeLoad<uint32_t>(in + 23) << 11, SafeLoad<uint32_t>(in + 23) >> 18 | SafeLoad<uint32_t>(in + 24) << 14, SafeLoad<uint32_t>(in + 24) >> 15 | SafeLoad<uint32_t>(in + 25) << 17 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 29-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 25) >> 12 | SafeLoad<uint32_t>(in + 26) << 20, SafeLoad<uint32_t>(in + 26) >> 9 | SafeLoad<uint32_t>(in + 27) << 23, SafeLoad<uint32_t>(in + 27) >> 6 | SafeLoad<uint32_t>(in + 28) << 26, SafeLoad<uint32_t>(in + 28) };
+ shifts = simd_batch{ 0, 0, 0, 3 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 29;
+ return in;
+}
+
+inline static const uint32_t* unpack30_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3fffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 30-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2) >> 26 | SafeLoad<uint32_t>(in + 3) << 6 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 30-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4) >> 22 | SafeLoad<uint32_t>(in + 5) << 10, SafeLoad<uint32_t>(in + 5) >> 20 | SafeLoad<uint32_t>(in + 6) << 12, SafeLoad<uint32_t>(in + 6) >> 18 | SafeLoad<uint32_t>(in + 7) << 14 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 30-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7) >> 16 | SafeLoad<uint32_t>(in + 8) << 16, SafeLoad<uint32_t>(in + 8) >> 14 | SafeLoad<uint32_t>(in + 9) << 18, SafeLoad<uint32_t>(in + 9) >> 12 | SafeLoad<uint32_t>(in + 10) << 20, SafeLoad<uint32_t>(in + 10) >> 10 | SafeLoad<uint32_t>(in + 11) << 22 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 30-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 11) >> 8 | SafeLoad<uint32_t>(in + 12) << 24, SafeLoad<uint32_t>(in + 12) >> 6 | SafeLoad<uint32_t>(in + 13) << 26, SafeLoad<uint32_t>(in + 13) >> 4 | SafeLoad<uint32_t>(in + 14) << 28, SafeLoad<uint32_t>(in + 14) };
+ shifts = simd_batch{ 0, 0, 0, 2 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 30-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 30 | SafeLoad<uint32_t>(in + 16) << 2, SafeLoad<uint32_t>(in + 16) >> 28 | SafeLoad<uint32_t>(in + 17) << 4, SafeLoad<uint32_t>(in + 17) >> 26 | SafeLoad<uint32_t>(in + 18) << 6 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 30-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 18) >> 24 | SafeLoad<uint32_t>(in + 19) << 8, SafeLoad<uint32_t>(in + 19) >> 22 | SafeLoad<uint32_t>(in + 20) << 10, SafeLoad<uint32_t>(in + 20) >> 20 | SafeLoad<uint32_t>(in + 21) << 12, SafeLoad<uint32_t>(in + 21) >> 18 | SafeLoad<uint32_t>(in + 22) << 14 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 30-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 22) >> 16 | SafeLoad<uint32_t>(in + 23) << 16, SafeLoad<uint32_t>(in + 23) >> 14 | SafeLoad<uint32_t>(in + 24) << 18, SafeLoad<uint32_t>(in + 24) >> 12 | SafeLoad<uint32_t>(in + 25) << 20, SafeLoad<uint32_t>(in + 25) >> 10 | SafeLoad<uint32_t>(in + 26) << 22 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 30-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 26) >> 8 | SafeLoad<uint32_t>(in + 27) << 24, SafeLoad<uint32_t>(in + 27) >> 6 | SafeLoad<uint32_t>(in + 28) << 26, SafeLoad<uint32_t>(in + 28) >> 4 | SafeLoad<uint32_t>(in + 29) << 28, SafeLoad<uint32_t>(in + 29) };
+ shifts = simd_batch{ 0, 0, 0, 2 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 30;
+ return in;
+}
+
+inline static const uint32_t* unpack31_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7fffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 31-bit bundles 0 to 3
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 31 | SafeLoad<uint32_t>(in + 1) << 1, SafeLoad<uint32_t>(in + 1) >> 30 | SafeLoad<uint32_t>(in + 2) << 2, SafeLoad<uint32_t>(in + 2) >> 29 | SafeLoad<uint32_t>(in + 3) << 3 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 31-bit bundles 4 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3) >> 28 | SafeLoad<uint32_t>(in + 4) << 4, SafeLoad<uint32_t>(in + 4) >> 27 | SafeLoad<uint32_t>(in + 5) << 5, SafeLoad<uint32_t>(in + 5) >> 26 | SafeLoad<uint32_t>(in + 6) << 6, SafeLoad<uint32_t>(in + 6) >> 25 | SafeLoad<uint32_t>(in + 7) << 7 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 31-bit bundles 8 to 11
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7) >> 24 | SafeLoad<uint32_t>(in + 8) << 8, SafeLoad<uint32_t>(in + 8) >> 23 | SafeLoad<uint32_t>(in + 9) << 9, SafeLoad<uint32_t>(in + 9) >> 22 | SafeLoad<uint32_t>(in + 10) << 10, SafeLoad<uint32_t>(in + 10) >> 21 | SafeLoad<uint32_t>(in + 11) << 11 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 31-bit bundles 12 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 11) >> 20 | SafeLoad<uint32_t>(in + 12) << 12, SafeLoad<uint32_t>(in + 12) >> 19 | SafeLoad<uint32_t>(in + 13) << 13, SafeLoad<uint32_t>(in + 13) >> 18 | SafeLoad<uint32_t>(in + 14) << 14, SafeLoad<uint32_t>(in + 14) >> 17 | SafeLoad<uint32_t>(in + 15) << 15 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 31-bit bundles 16 to 19
+ words = simd_batch{ SafeLoad<uint32_t>(in + 15) >> 16 | SafeLoad<uint32_t>(in + 16) << 16, SafeLoad<uint32_t>(in + 16) >> 15 | SafeLoad<uint32_t>(in + 17) << 17, SafeLoad<uint32_t>(in + 17) >> 14 | SafeLoad<uint32_t>(in + 18) << 18, SafeLoad<uint32_t>(in + 18) >> 13 | SafeLoad<uint32_t>(in + 19) << 19 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 31-bit bundles 20 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 19) >> 12 | SafeLoad<uint32_t>(in + 20) << 20, SafeLoad<uint32_t>(in + 20) >> 11 | SafeLoad<uint32_t>(in + 21) << 21, SafeLoad<uint32_t>(in + 21) >> 10 | SafeLoad<uint32_t>(in + 22) << 22, SafeLoad<uint32_t>(in + 22) >> 9 | SafeLoad<uint32_t>(in + 23) << 23 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 31-bit bundles 24 to 27
+ words = simd_batch{ SafeLoad<uint32_t>(in + 23) >> 8 | SafeLoad<uint32_t>(in + 24) << 24, SafeLoad<uint32_t>(in + 24) >> 7 | SafeLoad<uint32_t>(in + 25) << 25, SafeLoad<uint32_t>(in + 25) >> 6 | SafeLoad<uint32_t>(in + 26) << 26, SafeLoad<uint32_t>(in + 26) >> 5 | SafeLoad<uint32_t>(in + 27) << 27 };
+ shifts = simd_batch{ 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ // extract 31-bit bundles 28 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 27) >> 4 | SafeLoad<uint32_t>(in + 28) << 28, SafeLoad<uint32_t>(in + 28) >> 3 | SafeLoad<uint32_t>(in + 29) << 29, SafeLoad<uint32_t>(in + 29) >> 2 | SafeLoad<uint32_t>(in + 30) << 30, SafeLoad<uint32_t>(in + 30) };
+ shifts = simd_batch{ 0, 0, 0, 1 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 4;
+
+ in += 31;
+ return in;
+}
+
+inline static const uint32_t* unpack32_32(const uint32_t* in, uint32_t* out) {
+ memcpy(out, in, 32 * sizeof(*out));
+ in += 32;
+ out += 32;
+
+ return in;
+}
+
+}; // struct UnpackBits128
+
+} // namespace
+} // namespace internal
+} // namespace arrow
+
diff --git a/cpp/src/arrow/util/bpacking_simd256_generated.h b/cpp/src/arrow/util/bpacking_simd256_generated.h
new file mode 100644
index 0000000..a73bafe
--- /dev/null
+++ b/cpp/src/arrow/util/bpacking_simd256_generated.h
@@ -0,0 +1,1270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Automatically generated file; DO NOT EDIT.
+
+#pragma once
+
+#include <cstdint>
+#include <cstring>
+
+#include <xsimd/xsimd.hpp>
+
+#include "arrow/util/dispatch.h"
+#include "arrow/util/ubsan.h"
+
+namespace arrow {
+namespace internal {
+namespace {
+
+using ::arrow::util::SafeLoad;
+
+template <DispatchLevel level>
+struct UnpackBits256 {
+
+using simd_batch = xsimd::batch<uint32_t, 8>;
+
+inline static const uint32_t* unpack0_32(const uint32_t* in, uint32_t* out) {
+ memset(out, 0x0, 32 * sizeof(*out));
+ out += 32;
+
+ return in;
+}
+
+inline static const uint32_t* unpack1_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 1-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 0, 1, 2, 3, 4, 5, 6, 7 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 1-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 8, 9, 10, 11, 12, 13, 14, 15 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 1-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 16, 17, 18, 19, 20, 21, 22, 23 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 1-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 24, 25, 26, 27, 28, 29, 30, 31 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 1;
+ return in;
+}
+
+inline static const uint32_t* unpack2_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 2-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 0, 2, 4, 6, 8, 10, 12, 14 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 2-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 16, 18, 20, 22, 24, 26, 28, 30 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 2-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 2, 4, 6, 8, 10, 12, 14 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 2-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 16, 18, 20, 22, 24, 26, 28, 30 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 2;
+ return in;
+}
+
+inline static const uint32_t* unpack3_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 3-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 0, 3, 6, 9, 12, 15, 18, 21 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 3-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 24, 27, 0, 1, 4, 7, 10, 13 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 3-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 31 | SafeLoad<uint32_t>(in + 2) << 1, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 16, 19, 22, 25, 28, 0, 2, 5 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 3-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 8, 11, 14, 17, 20, 23, 26, 29 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 3;
+ return in;
+}
+
+inline static const uint32_t* unpack4_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xf;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 4-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 0, 4, 8, 12, 16, 20, 24, 28 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 4-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 4, 8, 12, 16, 20, 24, 28 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 4-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 0, 4, 8, 12, 16, 20, 24, 28 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 4-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 0, 4, 8, 12, 16, 20, 24, 28 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 4;
+ return in;
+}
+
+inline static const uint32_t* unpack5_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1f;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 5-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 5, 10, 15, 20, 25, 0, 3 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 5-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 8, 13, 18, 23, 0, 1, 6, 11 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 5-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 31 | SafeLoad<uint32_t>(in + 3) << 1, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 16, 21, 26, 0, 4, 9, 14, 19 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 5-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 29 | SafeLoad<uint32_t>(in + 4) << 3, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 24, 0, 2, 7, 12, 17, 22, 27 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 5;
+ return in;
+}
+
+inline static const uint32_t* unpack6_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3f;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 6-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 6, 12, 18, 24, 0, 4, 10 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 6-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 16, 22, 0, 2, 8, 14, 20, 26 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 6-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 0, 6, 12, 18, 24, 0, 4, 10 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 6-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 28 | SafeLoad<uint32_t>(in + 5) << 4, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 16, 22, 0, 2, 8, 14, 20, 26 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 6;
+ return in;
+}
+
+inline static const uint32_t* unpack7_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7f;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 7-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 28 | SafeLoad<uint32_t>(in + 1) << 4, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 7, 14, 21, 0, 3, 10, 17 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 7-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 31 | SafeLoad<uint32_t>(in + 2) << 1, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 27 | SafeLoad<uint32_t>(in + 3) << 5, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 24, 0, 6, 13, 20, 0, 2, 9 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 7-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 26 | SafeLoad<uint32_t>(in + 5) << 6, SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 16, 23, 0, 5, 12, 19, 0, 1 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 7-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 29 | SafeLoad<uint32_t>(in + 6) << 3, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) };
+ shifts = simd_batch{ 8, 15, 22, 0, 4, 11, 18, 25 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 7;
+ return in;
+}
+
+inline static const uint32_t* unpack8_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 8-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 8, 16, 24, 0, 8, 16, 24 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 8-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 0, 8, 16, 24, 0, 8, 16, 24 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 8-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 0, 8, 16, 24, 0, 8, 16, 24 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 8-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) };
+ shifts = simd_batch{ 0, 8, 16, 24, 0, 8, 16, 24 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 8;
+ return in;
+}
+
+inline static const uint32_t* unpack9_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1ff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 9-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 27 | SafeLoad<uint32_t>(in + 1) << 5, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 31 | SafeLoad<uint32_t>(in + 2) << 1 };
+ shifts = simd_batch{ 0, 9, 18, 0, 4, 13, 22, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 9-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 26 | SafeLoad<uint32_t>(in + 3) << 6, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2, SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 8, 17, 0, 3, 12, 21, 0, 7 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 9-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 25 | SafeLoad<uint32_t>(in + 5) << 7, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 29 | SafeLoad<uint32_t>(in + 6) << 3, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) };
+ shifts = simd_batch{ 16, 0, 2, 11, 20, 0, 6, 15 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 9-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6) >> 24 | SafeLoad<uint32_t>(in + 7) << 8, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) };
+ shifts = simd_batch{ 0, 1, 10, 19, 0, 5, 14, 23 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 9;
+ return in;
+}
+
+inline static const uint32_t* unpack10_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3ff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 10-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 0, 10, 20, 0, 8, 18, 0, 6 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 10-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 26 | SafeLoad<uint32_t>(in + 3) << 6, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 16, 0, 4, 14, 0, 2, 12, 22 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 10-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 30 | SafeLoad<uint32_t>(in + 6) << 2, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 28 | SafeLoad<uint32_t>(in + 7) << 4, SafeLoad<uint32_t>(in + 7) };
+ shifts = simd_batch{ 0, 10, 20, 0, 8, 18, 0, 6 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 10-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 26 | SafeLoad<uint32_t>(in + 8) << 6, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 24 | SafeLoad<uint32_t>(in + 9) << 8, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) };
+ shifts = simd_batch{ 16, 0, 4, 14, 0, 2, 12, 22 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 10;
+ return in;
+}
+
+inline static const uint32_t* unpack11_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7ff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 11-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 22 | SafeLoad<uint32_t>(in + 1) << 10, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 23 | SafeLoad<uint32_t>(in + 2) << 9, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 0, 11, 0, 1, 12, 0, 2, 13 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 11-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2) >> 24 | SafeLoad<uint32_t>(in + 3) << 8, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 25 | SafeLoad<uint32_t>(in + 4) << 7, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 26 | SafeLoad<uint32_t>(in + 5) << 6, SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 0, 3, 14, 0, 4, 15, 0, 5 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 11-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 27 | SafeLoad<uint32_t>(in + 6) << 5, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 28 | SafeLoad<uint32_t>(in + 7) << 4, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 29 | SafeLoad<uint32_t>(in + 8) << 3 };
+ shifts = simd_batch{ 16, 0, 6, 17, 0, 7, 18, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 11-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 30 | SafeLoad<uint32_t>(in + 9) << 2, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 31 | SafeLoad<uint32_t>(in + 10) << 1, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) };
+ shifts = simd_batch{ 8, 19, 0, 9, 20, 0, 10, 21 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 11;
+ return in;
+}
+
+inline static const uint32_t* unpack12_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xfff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 12-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 24 | SafeLoad<uint32_t>(in + 1) << 8, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 0, 12, 0, 4, 16, 0, 8, 20 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 12-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 28 | SafeLoad<uint32_t>(in + 5) << 4, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 0, 12, 0, 4, 16, 0, 8, 20 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 12-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 24 | SafeLoad<uint32_t>(in + 7) << 8, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) };
+ shifts = simd_batch{ 0, 12, 0, 4, 16, 0, 8, 20 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 12-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 24 | SafeLoad<uint32_t>(in + 10) << 8, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 28 | SafeLoad<uint32_t>(in + 11) << 4, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) };
+ shifts = simd_batch{ 0, 12, 0, 4, 16, 0, 8, 20 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 12;
+ return in;
+}
+
+inline static const uint32_t* unpack13_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1fff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 13-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 26 | SafeLoad<uint32_t>(in + 1) << 6, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 20 | SafeLoad<uint32_t>(in + 2) << 12, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 27 | SafeLoad<uint32_t>(in + 3) << 5 };
+ shifts = simd_batch{ 0, 13, 0, 7, 0, 1, 14, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 13-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 21 | SafeLoad<uint32_t>(in + 4) << 11, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 28 | SafeLoad<uint32_t>(in + 5) << 4, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 22 | SafeLoad<uint32_t>(in + 6) << 10, SafeLoad<uint32_t>(in + 6) };
+ shifts = simd_batch{ 8, 0, 2, 15, 0, 9, 0, 3 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 13-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 29 | SafeLoad<uint32_t>(in + 7) << 3, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 23 | SafeLoad<uint32_t>(in + 8) << 9, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 30 | SafeLoad<uint32_t>(in + 9) << 2, SafeLoad<uint32_t>(in + 9) };
+ shifts = simd_batch{ 16, 0, 10, 0, 4, 17, 0, 11 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 13-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 9) >> 24 | SafeLoad<uint32_t>(in + 10) << 8, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 31 | SafeLoad<uint32_t>(in + 11) << 1, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 25 | SafeLoad<uint32_t>(in + 12) << 7, SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) };
+ shifts = simd_batch{ 0, 5, 18, 0, 12, 0, 6, 19 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 13;
+ return in;
+}
+
+inline static const uint32_t* unpack14_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3fff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 14-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 28 | SafeLoad<uint32_t>(in + 1) << 4, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 24 | SafeLoad<uint32_t>(in + 2) << 8, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 20 | SafeLoad<uint32_t>(in + 3) << 12, SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 0, 14, 0, 10, 0, 6, 0, 2 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 14-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 26 | SafeLoad<uint32_t>(in + 5) << 6, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 22 | SafeLoad<uint32_t>(in + 6) << 10, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) };
+ shifts = simd_batch{ 16, 0, 12, 0, 8, 0, 4, 18 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 14-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 24 | SafeLoad<uint32_t>(in + 9) << 8, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 20 | SafeLoad<uint32_t>(in + 10) << 12, SafeLoad<uint32_t>(in + 10) };
+ shifts = simd_batch{ 0, 14, 0, 10, 0, 6, 0, 2 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 14-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 30 | SafeLoad<uint32_t>(in + 11) << 2, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 26 | SafeLoad<uint32_t>(in + 12) << 6, SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) >> 22 | SafeLoad<uint32_t>(in + 13) << 10, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) };
+ shifts = simd_batch{ 16, 0, 12, 0, 8, 0, 4, 18 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 14;
+ return in;
+}
+
+inline static const uint32_t* unpack15_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7fff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 15-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 26 | SafeLoad<uint32_t>(in + 3) << 6, SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 0, 15, 0, 13, 0, 11, 0, 9 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 15-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 22 | SafeLoad<uint32_t>(in + 5) << 10, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 20 | SafeLoad<uint32_t>(in + 6) << 12, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 18 | SafeLoad<uint32_t>(in + 7) << 14, SafeLoad<uint32_t>(in + 7) };
+ shifts = simd_batch{ 0, 7, 0, 5, 0, 3, 0, 1 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 15-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 31 | SafeLoad<uint32_t>(in + 8) << 1, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 29 | SafeLoad<uint32_t>(in + 9) << 3, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 27 | SafeLoad<uint32_t>(in + 10) << 5, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 25 | SafeLoad<uint32_t>(in + 11) << 7 };
+ shifts = simd_batch{ 16, 0, 14, 0, 12, 0, 10, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 15-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 23 | SafeLoad<uint32_t>(in + 12) << 9, SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) >> 21 | SafeLoad<uint32_t>(in + 13) << 11, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 19 | SafeLoad<uint32_t>(in + 14) << 13, SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) };
+ shifts = simd_batch{ 8, 0, 6, 0, 4, 0, 2, 17 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 15;
+ return in;
+}
+
+inline static const uint32_t* unpack16_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 16-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 0, 16, 0, 16, 0, 16, 0, 16 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 16-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) };
+ shifts = simd_batch{ 0, 16, 0, 16, 0, 16, 0, 16 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 16-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) };
+ shifts = simd_batch{ 0, 16, 0, 16, 0, 16, 0, 16 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 16-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) };
+ shifts = simd_batch{ 0, 16, 0, 16, 0, 16, 0, 16 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 16;
+ return in;
+}
+
+inline static const uint32_t* unpack17_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1ffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 17-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 17 | SafeLoad<uint32_t>(in + 1) << 15, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 19 | SafeLoad<uint32_t>(in + 2) << 13, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 21 | SafeLoad<uint32_t>(in + 3) << 11, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 23 | SafeLoad<uint32_t>(in + 4) << 9 };
+ shifts = simd_batch{ 0, 0, 2, 0, 4, 0, 6, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 17-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 25 | SafeLoad<uint32_t>(in + 5) << 7, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 27 | SafeLoad<uint32_t>(in + 6) << 5, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 29 | SafeLoad<uint32_t>(in + 7) << 3, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 31 | SafeLoad<uint32_t>(in + 8) << 1 };
+ shifts = simd_batch{ 8, 0, 10, 0, 12, 0, 14, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 17-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 8) >> 16 | SafeLoad<uint32_t>(in + 9) << 16, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 18 | SafeLoad<uint32_t>(in + 10) << 14, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 20 | SafeLoad<uint32_t>(in + 11) << 12, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 22 | SafeLoad<uint32_t>(in + 12) << 10, SafeLoad<uint32_t>(in + 12) };
+ shifts = simd_batch{ 0, 1, 0, 3, 0, 5, 0, 7 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 17-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 12) >> 24 | SafeLoad<uint32_t>(in + 13) << 8, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 26 | SafeLoad<uint32_t>(in + 14) << 6, SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) >> 28 | SafeLoad<uint32_t>(in + 15) << 4, SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 30 | SafeLoad<uint32_t>(in + 16) << 2, SafeLoad<uint32_t>(in + 16) };
+ shifts = simd_batch{ 0, 9, 0, 11, 0, 13, 0, 15 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 17;
+ return in;
+}
+
+inline static const uint32_t* unpack18_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3ffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 18-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 18 | SafeLoad<uint32_t>(in + 1) << 14, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 22 | SafeLoad<uint32_t>(in + 2) << 10, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 26 | SafeLoad<uint32_t>(in + 3) << 6, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2 };
+ shifts = simd_batch{ 0, 0, 4, 0, 8, 0, 12, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 18-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4) >> 16 | SafeLoad<uint32_t>(in + 5) << 16, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 20 | SafeLoad<uint32_t>(in + 6) << 12, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 24 | SafeLoad<uint32_t>(in + 7) << 8, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8) };
+ shifts = simd_batch{ 0, 2, 0, 6, 0, 10, 0, 14 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 18-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 18 | SafeLoad<uint32_t>(in + 10) << 14, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 22 | SafeLoad<uint32_t>(in + 11) << 10, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 26 | SafeLoad<uint32_t>(in + 12) << 6, SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) >> 30 | SafeLoad<uint32_t>(in + 13) << 2 };
+ shifts = simd_batch{ 0, 0, 4, 0, 8, 0, 12, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 18-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 13) >> 16 | SafeLoad<uint32_t>(in + 14) << 16, SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) >> 20 | SafeLoad<uint32_t>(in + 15) << 12, SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 24 | SafeLoad<uint32_t>(in + 16) << 8, SafeLoad<uint32_t>(in + 16), SafeLoad<uint32_t>(in + 16) >> 28 | SafeLoad<uint32_t>(in + 17) << 4, SafeLoad<uint32_t>(in + 17) };
+ shifts = simd_batch{ 0, 2, 0, 6, 0, 10, 0, 14 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 18;
+ return in;
+}
+
+inline static const uint32_t* unpack19_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7ffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 19-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 19 | SafeLoad<uint32_t>(in + 1) << 13, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 25 | SafeLoad<uint32_t>(in + 2) << 7, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 31 | SafeLoad<uint32_t>(in + 3) << 1, SafeLoad<uint32_t>(in + 3) >> 18 | SafeLoad<uint32_t>(in + 4) << 14, SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 0, 0, 6, 0, 12, 0, 0, 5 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 19-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4) >> 24 | SafeLoad<uint32_t>(in + 5) << 8, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 30 | SafeLoad<uint32_t>(in + 6) << 2, SafeLoad<uint32_t>(in + 6) >> 17 | SafeLoad<uint32_t>(in + 7) << 15, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 23 | SafeLoad<uint32_t>(in + 8) << 9, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 29 | SafeLoad<uint32_t>(in + 9) << 3 };
+ shifts = simd_batch{ 0, 11, 0, 0, 4, 0, 10, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 19-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 9) >> 16 | SafeLoad<uint32_t>(in + 10) << 16, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 22 | SafeLoad<uint32_t>(in + 11) << 10, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 28 | SafeLoad<uint32_t>(in + 12) << 4, SafeLoad<uint32_t>(in + 12) >> 15 | SafeLoad<uint32_t>(in + 13) << 17, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 21 | SafeLoad<uint32_t>(in + 14) << 11 };
+ shifts = simd_batch{ 0, 3, 0, 9, 0, 0, 2, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 19-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) >> 27 | SafeLoad<uint32_t>(in + 15) << 5, SafeLoad<uint32_t>(in + 15) >> 14 | SafeLoad<uint32_t>(in + 16) << 18, SafeLoad<uint32_t>(in + 16), SafeLoad<uint32_t>(in + 16) >> 20 | SafeLoad<uint32_t>(in + 17) << 12, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 26 | SafeLoad<uint32_t>(in + 18) << 6, SafeLoad<uint32_t>(in + 18) };
+ shifts = simd_batch{ 8, 0, 0, 1, 0, 7, 0, 13 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 19;
+ return in;
+}
+
+inline static const uint32_t* unpack20_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xfffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 20-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 20 | SafeLoad<uint32_t>(in + 1) << 12, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2) >> 16 | SafeLoad<uint32_t>(in + 3) << 16, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 0, 0, 8, 0, 0, 4, 0, 12 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 20-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 20 | SafeLoad<uint32_t>(in + 6) << 12, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 28 | SafeLoad<uint32_t>(in + 7) << 4, SafeLoad<uint32_t>(in + 7) >> 16 | SafeLoad<uint32_t>(in + 8) << 16, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 24 | SafeLoad<uint32_t>(in + 9) << 8, SafeLoad<uint32_t>(in + 9) };
+ shifts = simd_batch{ 0, 0, 8, 0, 0, 4, 0, 12 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 20-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 20 | SafeLoad<uint32_t>(in + 11) << 12, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 28 | SafeLoad<uint32_t>(in + 12) << 4, SafeLoad<uint32_t>(in + 12) >> 16 | SafeLoad<uint32_t>(in + 13) << 16, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 24 | SafeLoad<uint32_t>(in + 14) << 8, SafeLoad<uint32_t>(in + 14) };
+ shifts = simd_batch{ 0, 0, 8, 0, 0, 4, 0, 12 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 20-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 20 | SafeLoad<uint32_t>(in + 16) << 12, SafeLoad<uint32_t>(in + 16), SafeLoad<uint32_t>(in + 16) >> 28 | SafeLoad<uint32_t>(in + 17) << 4, SafeLoad<uint32_t>(in + 17) >> 16 | SafeLoad<uint32_t>(in + 18) << 16, SafeLoad<uint32_t>(in + 18), SafeLoad<uint32_t>(in + 18) >> 24 | SafeLoad<uint32_t>(in + 19) << 8, SafeLoad<uint32_t>(in + 19) };
+ shifts = simd_batch{ 0, 0, 8, 0, 0, 4, 0, 12 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 20;
+ return in;
+}
+
+inline static const uint32_t* unpack21_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1fffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 21-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 21 | SafeLoad<uint32_t>(in + 1) << 11, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 31 | SafeLoad<uint32_t>(in + 2) << 1, SafeLoad<uint32_t>(in + 2) >> 20 | SafeLoad<uint32_t>(in + 3) << 12, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2, SafeLoad<uint32_t>(in + 4) >> 19 | SafeLoad<uint32_t>(in + 5) << 13 };
+ shifts = simd_batch{ 0, 0, 10, 0, 0, 9, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 21-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 29 | SafeLoad<uint32_t>(in + 6) << 3, SafeLoad<uint32_t>(in + 6) >> 18 | SafeLoad<uint32_t>(in + 7) << 14, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8) >> 17 | SafeLoad<uint32_t>(in + 9) << 15, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 27 | SafeLoad<uint32_t>(in + 10) << 5 };
+ shifts = simd_batch{ 8, 0, 0, 7, 0, 0, 6, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 21-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 10) >> 16 | SafeLoad<uint32_t>(in + 11) << 16, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 26 | SafeLoad<uint32_t>(in + 12) << 6, SafeLoad<uint32_t>(in + 12) >> 15 | SafeLoad<uint32_t>(in + 13) << 17, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 25 | SafeLoad<uint32_t>(in + 14) << 7, SafeLoad<uint32_t>(in + 14) >> 14 | SafeLoad<uint32_t>(in + 15) << 18, SafeLoad<uint32_t>(in + 15) };
+ shifts = simd_batch{ 0, 5, 0, 0, 4, 0, 0, 3 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 21-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 15) >> 24 | SafeLoad<uint32_t>(in + 16) << 8, SafeLoad<uint32_t>(in + 16) >> 13 | SafeLoad<uint32_t>(in + 17) << 19, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 23 | SafeLoad<uint32_t>(in + 18) << 9, SafeLoad<uint32_t>(in + 18) >> 12 | SafeLoad<uint32_t>(in + 19) << 20, SafeLoad<uint32_t>(in + 19), SafeLoad<uint32_t>(in + 19) >> 22 | SafeLoad<uint32_t>(in + 20) << 10, SafeLoad<uint32_t>(in + 20) };
+ shifts = simd_batch{ 0, 0, 2, 0, 0, 1, 0, 11 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 21;
+ return in;
+}
+
+inline static const uint32_t* unpack22_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3fffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 22-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 22 | SafeLoad<uint32_t>(in + 1) << 10, SafeLoad<uint32_t>(in + 1) >> 12 | SafeLoad<uint32_t>(in + 2) << 20, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 24 | SafeLoad<uint32_t>(in + 3) << 8, SafeLoad<uint32_t>(in + 3) >> 14 | SafeLoad<uint32_t>(in + 4) << 18, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 26 | SafeLoad<uint32_t>(in + 5) << 6 };
+ shifts = simd_batch{ 0, 0, 0, 2, 0, 0, 4, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 22-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5) >> 16 | SafeLoad<uint32_t>(in + 6) << 16, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 28 | SafeLoad<uint32_t>(in + 7) << 4, SafeLoad<uint32_t>(in + 7) >> 18 | SafeLoad<uint32_t>(in + 8) << 14, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 30 | SafeLoad<uint32_t>(in + 9) << 2, SafeLoad<uint32_t>(in + 9) >> 20 | SafeLoad<uint32_t>(in + 10) << 12, SafeLoad<uint32_t>(in + 10) };
+ shifts = simd_batch{ 0, 6, 0, 0, 8, 0, 0, 10 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 22-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 22 | SafeLoad<uint32_t>(in + 12) << 10, SafeLoad<uint32_t>(in + 12) >> 12 | SafeLoad<uint32_t>(in + 13) << 20, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 24 | SafeLoad<uint32_t>(in + 14) << 8, SafeLoad<uint32_t>(in + 14) >> 14 | SafeLoad<uint32_t>(in + 15) << 18, SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 26 | SafeLoad<uint32_t>(in + 16) << 6 };
+ shifts = simd_batch{ 0, 0, 0, 2, 0, 0, 4, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 22-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 16) >> 16 | SafeLoad<uint32_t>(in + 17) << 16, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 28 | SafeLoad<uint32_t>(in + 18) << 4, SafeLoad<uint32_t>(in + 18) >> 18 | SafeLoad<uint32_t>(in + 19) << 14, SafeLoad<uint32_t>(in + 19), SafeLoad<uint32_t>(in + 19) >> 30 | SafeLoad<uint32_t>(in + 20) << 2, SafeLoad<uint32_t>(in + 20) >> 20 | SafeLoad<uint32_t>(in + 21) << 12, SafeLoad<uint32_t>(in + 21) };
+ shifts = simd_batch{ 0, 6, 0, 0, 8, 0, 0, 10 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 22;
+ return in;
+}
+
+inline static const uint32_t* unpack23_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7fffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 23-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 23 | SafeLoad<uint32_t>(in + 1) << 9, SafeLoad<uint32_t>(in + 1) >> 14 | SafeLoad<uint32_t>(in + 2) << 18, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 28 | SafeLoad<uint32_t>(in + 3) << 4, SafeLoad<uint32_t>(in + 3) >> 19 | SafeLoad<uint32_t>(in + 4) << 13, SafeLoad<uint32_t>(in + 4) >> 10 | SafeLoad<uint32_t>(in + 5) << 22, SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 0, 0, 0, 5, 0, 0, 0, 1 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 23-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5) >> 24 | SafeLoad<uint32_t>(in + 6) << 8, SafeLoad<uint32_t>(in + 6) >> 15 | SafeLoad<uint32_t>(in + 7) << 17, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 29 | SafeLoad<uint32_t>(in + 8) << 3, SafeLoad<uint32_t>(in + 8) >> 20 | SafeLoad<uint32_t>(in + 9) << 12, SafeLoad<uint32_t>(in + 9) >> 11 | SafeLoad<uint32_t>(in + 10) << 21, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 25 | SafeLoad<uint32_t>(in + 11) << 7 };
+ shifts = simd_batch{ 0, 0, 6, 0, 0, 0, 2, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 23-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 11) >> 16 | SafeLoad<uint32_t>(in + 12) << 16, SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) >> 30 | SafeLoad<uint32_t>(in + 13) << 2, SafeLoad<uint32_t>(in + 13) >> 21 | SafeLoad<uint32_t>(in + 14) << 11, SafeLoad<uint32_t>(in + 14) >> 12 | SafeLoad<uint32_t>(in + 15) << 20, SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 26 | SafeLoad<uint32_t>(in + 16) << 6, SafeLoad<uint32_t>(in + 16) >> 17 | SafeLoad<uint32_t>(in + 17) << 15 };
+ shifts = simd_batch{ 0, 7, 0, 0, 0, 3, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 23-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 31 | SafeLoad<uint32_t>(in + 18) << 1, SafeLoad<uint32_t>(in + 18) >> 22 | SafeLoad<uint32_t>(in + 19) << 10, SafeLoad<uint32_t>(in + 19) >> 13 | SafeLoad<uint32_t>(in + 20) << 19, SafeLoad<uint32_t>(in + 20), SafeLoad<uint32_t>(in + 20) >> 27 | SafeLoad<uint32_t>(in + 21) << 5, SafeLoad<uint32_t>(in + 21) >> 18 | SafeLoad<uint32_t>(in + 22) << 14, SafeLoad<uint32_t>(in + 22) };
+ shifts = simd_batch{ 8, 0, 0, 0, 4, 0, 0, 9 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 23;
+ return in;
+}
+
+inline static const uint32_t* unpack24_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 24-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 24 | SafeLoad<uint32_t>(in + 1) << 8, SafeLoad<uint32_t>(in + 1) >> 16 | SafeLoad<uint32_t>(in + 2) << 16, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4) >> 16 | SafeLoad<uint32_t>(in + 5) << 16, SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 0, 0, 0, 8, 0, 0, 0, 8 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 24-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 24 | SafeLoad<uint32_t>(in + 7) << 8, SafeLoad<uint32_t>(in + 7) >> 16 | SafeLoad<uint32_t>(in + 8) << 16, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 24 | SafeLoad<uint32_t>(in + 10) << 8, SafeLoad<uint32_t>(in + 10) >> 16 | SafeLoad<uint32_t>(in + 11) << 16, SafeLoad<uint32_t>(in + 11) };
+ shifts = simd_batch{ 0, 0, 0, 8, 0, 0, 0, 8 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 24-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) >> 24 | SafeLoad<uint32_t>(in + 13) << 8, SafeLoad<uint32_t>(in + 13) >> 16 | SafeLoad<uint32_t>(in + 14) << 16, SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 24 | SafeLoad<uint32_t>(in + 16) << 8, SafeLoad<uint32_t>(in + 16) >> 16 | SafeLoad<uint32_t>(in + 17) << 16, SafeLoad<uint32_t>(in + 17) };
+ shifts = simd_batch{ 0, 0, 0, 8, 0, 0, 0, 8 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 24-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 18), SafeLoad<uint32_t>(in + 18) >> 24 | SafeLoad<uint32_t>(in + 19) << 8, SafeLoad<uint32_t>(in + 19) >> 16 | SafeLoad<uint32_t>(in + 20) << 16, SafeLoad<uint32_t>(in + 20), SafeLoad<uint32_t>(in + 21), SafeLoad<uint32_t>(in + 21) >> 24 | SafeLoad<uint32_t>(in + 22) << 8, SafeLoad<uint32_t>(in + 22) >> 16 | SafeLoad<uint32_t>(in + 23) << 16, SafeLoad<uint32_t>(in + 23) };
+ shifts = simd_batch{ 0, 0, 0, 8, 0, 0, 0, 8 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 24;
+ return in;
+}
+
+inline static const uint32_t* unpack25_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1ffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 25-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 25 | SafeLoad<uint32_t>(in + 1) << 7, SafeLoad<uint32_t>(in + 1) >> 18 | SafeLoad<uint32_t>(in + 2) << 14, SafeLoad<uint32_t>(in + 2) >> 11 | SafeLoad<uint32_t>(in + 3) << 21, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 29 | SafeLoad<uint32_t>(in + 4) << 3, SafeLoad<uint32_t>(in + 4) >> 22 | SafeLoad<uint32_t>(in + 5) << 10, SafeLoad<uint32_t>(in + 5) >> 15 | SafeLoad<uint32_t>(in + 6) << 17 };
+ shifts = simd_batch{ 0, 0, 0, 0, 4, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 25-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6) >> 8 | SafeLoad<uint32_t>(in + 7) << 24, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 26 | SafeLoad<uint32_t>(in + 8) << 6, SafeLoad<uint32_t>(in + 8) >> 19 | SafeLoad<uint32_t>(in + 9) << 13, SafeLoad<uint32_t>(in + 9) >> 12 | SafeLoad<uint32_t>(in + 10) << 20, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 30 | SafeLoad<uint32_t>(in + 11) << 2, SafeLoad<uint32_t>(in + 11) >> 23 | SafeLoad<uint32_t>(in + 12) << 9 };
+ shifts = simd_batch{ 0, 1, 0, 0, 0, 5, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 25-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 12) >> 16 | SafeLoad<uint32_t>(in + 13) << 16, SafeLoad<uint32_t>(in + 13) >> 9 | SafeLoad<uint32_t>(in + 14) << 23, SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) >> 27 | SafeLoad<uint32_t>(in + 15) << 5, SafeLoad<uint32_t>(in + 15) >> 20 | SafeLoad<uint32_t>(in + 16) << 12, SafeLoad<uint32_t>(in + 16) >> 13 | SafeLoad<uint32_t>(in + 17) << 19, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 31 | SafeLoad<uint32_t>(in + 18) << 1 };
+ shifts = simd_batch{ 0, 0, 2, 0, 0, 0, 6, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 25-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 18) >> 24 | SafeLoad<uint32_t>(in + 19) << 8, SafeLoad<uint32_t>(in + 19) >> 17 | SafeLoad<uint32_t>(in + 20) << 15, SafeLoad<uint32_t>(in + 20) >> 10 | SafeLoad<uint32_t>(in + 21) << 22, SafeLoad<uint32_t>(in + 21), SafeLoad<uint32_t>(in + 21) >> 28 | SafeLoad<uint32_t>(in + 22) << 4, SafeLoad<uint32_t>(in + 22) >> 21 | SafeLoad<uint32_t>(in + 23) << 11, SafeLoad<uint32_t>(in + 23) >> 14 | SafeLoad<uint32_t>(in + 24) << 18, SafeLoad<uint32_t>(in + 24) };
+ shifts = simd_batch{ 0, 0, 0, 3, 0, 0, 0, 7 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 25;
+ return in;
+}
+
+inline static const uint32_t* unpack26_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3ffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 26-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 26 | SafeLoad<uint32_t>(in + 1) << 6, SafeLoad<uint32_t>(in + 1) >> 20 | SafeLoad<uint32_t>(in + 2) << 12, SafeLoad<uint32_t>(in + 2) >> 14 | SafeLoad<uint32_t>(in + 3) << 18, SafeLoad<uint32_t>(in + 3) >> 8 | SafeLoad<uint32_t>(in + 4) << 24, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 28 | SafeLoad<uint32_t>(in + 5) << 4, SafeLoad<uint32_t>(in + 5) >> 22 | SafeLoad<uint32_t>(in + 6) << 10 };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 2, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 26-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6) >> 16 | SafeLoad<uint32_t>(in + 7) << 16, SafeLoad<uint32_t>(in + 7) >> 10 | SafeLoad<uint32_t>(in + 8) << 22, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 30 | SafeLoad<uint32_t>(in + 9) << 2, SafeLoad<uint32_t>(in + 9) >> 24 | SafeLoad<uint32_t>(in + 10) << 8, SafeLoad<uint32_t>(in + 10) >> 18 | SafeLoad<uint32_t>(in + 11) << 14, SafeLoad<uint32_t>(in + 11) >> 12 | SafeLoad<uint32_t>(in + 12) << 20, SafeLoad<uint32_t>(in + 12) };
+ shifts = simd_batch{ 0, 0, 4, 0, 0, 0, 0, 6 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 26-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 26 | SafeLoad<uint32_t>(in + 14) << 6, SafeLoad<uint32_t>(in + 14) >> 20 | SafeLoad<uint32_t>(in + 15) << 12, SafeLoad<uint32_t>(in + 15) >> 14 | SafeLoad<uint32_t>(in + 16) << 18, SafeLoad<uint32_t>(in + 16) >> 8 | SafeLoad<uint32_t>(in + 17) << 24, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 28 | SafeLoad<uint32_t>(in + 18) << 4, SafeLoad<uint32_t>(in + 18) >> 22 | SafeLoad<uint32_t>(in + 19) << 10 };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 2, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 26-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 19) >> 16 | SafeLoad<uint32_t>(in + 20) << 16, SafeLoad<uint32_t>(in + 20) >> 10 | SafeLoad<uint32_t>(in + 21) << 22, SafeLoad<uint32_t>(in + 21), SafeLoad<uint32_t>(in + 21) >> 30 | SafeLoad<uint32_t>(in + 22) << 2, SafeLoad<uint32_t>(in + 22) >> 24 | SafeLoad<uint32_t>(in + 23) << 8, SafeLoad<uint32_t>(in + 23) >> 18 | SafeLoad<uint32_t>(in + 24) << 14, SafeLoad<uint32_t>(in + 24) >> 12 | SafeLoad<uint32_t>(in + 25) << 20, SafeLoad<uint32_t>(in + 25) };
+ shifts = simd_batch{ 0, 0, 4, 0, 0, 0, 0, 6 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 26;
+ return in;
+}
+
+inline static const uint32_t* unpack27_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7ffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 27-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 27 | SafeLoad<uint32_t>(in + 1) << 5, SafeLoad<uint32_t>(in + 1) >> 22 | SafeLoad<uint32_t>(in + 2) << 10, SafeLoad<uint32_t>(in + 2) >> 17 | SafeLoad<uint32_t>(in + 3) << 15, SafeLoad<uint32_t>(in + 3) >> 12 | SafeLoad<uint32_t>(in + 4) << 20, SafeLoad<uint32_t>(in + 4) >> 7 | SafeLoad<uint32_t>(in + 5) << 25, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 29 | SafeLoad<uint32_t>(in + 6) << 3 };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 2, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 27-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6) >> 24 | SafeLoad<uint32_t>(in + 7) << 8, SafeLoad<uint32_t>(in + 7) >> 19 | SafeLoad<uint32_t>(in + 8) << 13, SafeLoad<uint32_t>(in + 8) >> 14 | SafeLoad<uint32_t>(in + 9) << 18, SafeLoad<uint32_t>(in + 9) >> 9 | SafeLoad<uint32_t>(in + 10) << 23, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 31 | SafeLoad<uint32_t>(in + 11) << 1, SafeLoad<uint32_t>(in + 11) >> 26 | SafeLoad<uint32_t>(in + 12) << 6, SafeLoad<uint32_t>(in + 12) >> 21 | SafeLoad<uint32_t>(in + 13) << 11 };
+ shifts = simd_batch{ 0, 0, 0, 0, 4, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 27-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 13) >> 16 | SafeLoad<uint32_t>(in + 14) << 16, SafeLoad<uint32_t>(in + 14) >> 11 | SafeLoad<uint32_t>(in + 15) << 21, SafeLoad<uint32_t>(in + 15) >> 6 | SafeLoad<uint32_t>(in + 16) << 26, SafeLoad<uint32_t>(in + 16), SafeLoad<uint32_t>(in + 16) >> 28 | SafeLoad<uint32_t>(in + 17) << 4, SafeLoad<uint32_t>(in + 17) >> 23 | SafeLoad<uint32_t>(in + 18) << 9, SafeLoad<uint32_t>(in + 18) >> 18 | SafeLoad<uint32_t>(in + 19) << 14, SafeLoad<uint32_t>(in + 19) >> 13 | SafeLoad<uint32_t>(in + 20) << 19 };
+ shifts = simd_batch{ 0, 0, 0, 1, 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 27-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 20) >> 8 | SafeLoad<uint32_t>(in + 21) << 24, SafeLoad<uint32_t>(in + 21), SafeLoad<uint32_t>(in + 21) >> 30 | SafeLoad<uint32_t>(in + 22) << 2, SafeLoad<uint32_t>(in + 22) >> 25 | SafeLoad<uint32_t>(in + 23) << 7, SafeLoad<uint32_t>(in + 23) >> 20 | SafeLoad<uint32_t>(in + 24) << 12, SafeLoad<uint32_t>(in + 24) >> 15 | SafeLoad<uint32_t>(in + 25) << 17, SafeLoad<uint32_t>(in + 25) >> 10 | SafeLoad<uint32_t>(in + 26) << 22, SafeLoad<uint32_t>(in + 26) };
+ shifts = simd_batch{ 0, 3, 0, 0, 0, 0, 0, 5 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 27;
+ return in;
+}
+
+inline static const uint32_t* unpack28_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xfffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 28-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 28 | SafeLoad<uint32_t>(in + 1) << 4, SafeLoad<uint32_t>(in + 1) >> 24 | SafeLoad<uint32_t>(in + 2) << 8, SafeLoad<uint32_t>(in + 2) >> 20 | SafeLoad<uint32_t>(in + 3) << 12, SafeLoad<uint32_t>(in + 3) >> 16 | SafeLoad<uint32_t>(in + 4) << 16, SafeLoad<uint32_t>(in + 4) >> 12 | SafeLoad<uint32_t>(in + 5) << 20, SafeLoad<uint32_t>(in + 5) >> 8 | SafeLoad<uint32_t>(in + 6) << 24, SafeLoad<uint32_t>(in + 6) };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 4 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 28-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8) >> 24 | SafeLoad<uint32_t>(in + 9) << 8, SafeLoad<uint32_t>(in + 9) >> 20 | SafeLoad<uint32_t>(in + 10) << 12, SafeLoad<uint32_t>(in + 10) >> 16 | SafeLoad<uint32_t>(in + 11) << 16, SafeLoad<uint32_t>(in + 11) >> 12 | SafeLoad<uint32_t>(in + 12) << 20, SafeLoad<uint32_t>(in + 12) >> 8 | SafeLoad<uint32_t>(in + 13) << 24, SafeLoad<uint32_t>(in + 13) };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 4 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 28-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) >> 28 | SafeLoad<uint32_t>(in + 15) << 4, SafeLoad<uint32_t>(in + 15) >> 24 | SafeLoad<uint32_t>(in + 16) << 8, SafeLoad<uint32_t>(in + 16) >> 20 | SafeLoad<uint32_t>(in + 17) << 12, SafeLoad<uint32_t>(in + 17) >> 16 | SafeLoad<uint32_t>(in + 18) << 16, SafeLoad<uint32_t>(in + 18) >> 12 | SafeLoad<uint32_t>(in + 19) << 20, SafeLoad<uint32_t>(in + 19) >> 8 | SafeLoad<uint32_t>(in + 20) << 24, SafeLoad<uint32_t>(in + 20) };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 4 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 28-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 21), SafeLoad<uint32_t>(in + 21) >> 28 | SafeLoad<uint32_t>(in + 22) << 4, SafeLoad<uint32_t>(in + 22) >> 24 | SafeLoad<uint32_t>(in + 23) << 8, SafeLoad<uint32_t>(in + 23) >> 20 | SafeLoad<uint32_t>(in + 24) << 12, SafeLoad<uint32_t>(in + 24) >> 16 | SafeLoad<uint32_t>(in + 25) << 16, SafeLoad<uint32_t>(in + 25) >> 12 | SafeLoad<uint32_t>(in + 26) << 20, SafeLoad<uint32_t>(in + 26) >> 8 | SafeLoad<uint32_t>(in + 27) << 24, SafeLoad<uint32_t>(in + 27) };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 4 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 28;
+ return in;
+}
+
+inline static const uint32_t* unpack29_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1fffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 29-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 29 | SafeLoad<uint32_t>(in + 1) << 3, SafeLoad<uint32_t>(in + 1) >> 26 | SafeLoad<uint32_t>(in + 2) << 6, SafeLoad<uint32_t>(in + 2) >> 23 | SafeLoad<uint32_t>(in + 3) << 9, SafeLoad<uint32_t>(in + 3) >> 20 | SafeLoad<uint32_t>(in + 4) << 12, SafeLoad<uint32_t>(in + 4) >> 17 | SafeLoad<uint32_t>(in + 5) << 15, SafeLoad<uint32_t>(in + 5) >> 14 | SafeLoad<uint32_t>(in + 6) << 18, SafeLoad<uint32_t>(in + 6) >> 11 | SafeLoad<uint32_t>(in + 7) << 21 };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 29-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7) >> 8 | SafeLoad<uint32_t>(in + 8) << 24, SafeLoad<uint32_t>(in + 8) >> 5 | SafeLoad<uint32_t>(in + 9) << 27, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 31 | SafeLoad<uint32_t>(in + 10) << 1, SafeLoad<uint32_t>(in + 10) >> 28 | SafeLoad<uint32_t>(in + 11) << 4, SafeLoad<uint32_t>(in + 11) >> 25 | SafeLoad<uint32_t>(in + 12) << 7, SafeLoad<uint32_t>(in + 12) >> 22 | SafeLoad<uint32_t>(in + 13) << 10, SafeLoad<uint32_t>(in + 13) >> 19 | SafeLoad<uint32_t>(in + 14) << 13 };
+ shifts = simd_batch{ 0, 0, 2, 0, 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 29-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 14) >> 16 | SafeLoad<uint32_t>(in + 15) << 16, SafeLoad<uint32_t>(in + 15) >> 13 | SafeLoad<uint32_t>(in + 16) << 19, SafeLoad<uint32_t>(in + 16) >> 10 | SafeLoad<uint32_t>(in + 17) << 22, SafeLoad<uint32_t>(in + 17) >> 7 | SafeLoad<uint32_t>(in + 18) << 25, SafeLoad<uint32_t>(in + 18) >> 4 | SafeLoad<uint32_t>(in + 19) << 28, SafeLoad<uint32_t>(in + 19), SafeLoad<uint32_t>(in + 19) >> 30 | SafeLoad<uint32_t>(in + 20) << 2, SafeLoad<uint32_t>(in + 20) >> 27 | SafeLoad<uint32_t>(in + 21) << 5 };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 1, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 29-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 21) >> 24 | SafeLoad<uint32_t>(in + 22) << 8, SafeLoad<uint32_t>(in + 22) >> 21 | SafeLoad<uint32_t>(in + 23) << 11, SafeLoad<uint32_t>(in + 23) >> 18 | SafeLoad<uint32_t>(in + 24) << 14, SafeLoad<uint32_t>(in + 24) >> 15 | SafeLoad<uint32_t>(in + 25) << 17, SafeLoad<uint32_t>(in + 25) >> 12 | SafeLoad<uint32_t>(in + 26) << 20, SafeLoad<uint32_t>(in + 26) >> 9 | SafeLoad<uint32_t>(in + 27) << 23, SafeLoad<uint32_t>(in + 27) >> 6 | SafeLoad<uint32_t>(in + 28) << 26, SafeLoad<uint32_t>(in + 28) };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 3 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 29;
+ return in;
+}
+
+inline static const uint32_t* unpack30_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3fffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 30-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2) >> 26 | SafeLoad<uint32_t>(in + 3) << 6, SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4) >> 22 | SafeLoad<uint32_t>(in + 5) << 10, SafeLoad<uint32_t>(in + 5) >> 20 | SafeLoad<uint32_t>(in + 6) << 12, SafeLoad<uint32_t>(in + 6) >> 18 | SafeLoad<uint32_t>(in + 7) << 14 };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 30-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7) >> 16 | SafeLoad<uint32_t>(in + 8) << 16, SafeLoad<uint32_t>(in + 8) >> 14 | SafeLoad<uint32_t>(in + 9) << 18, SafeLoad<uint32_t>(in + 9) >> 12 | SafeLoad<uint32_t>(in + 10) << 20, SafeLoad<uint32_t>(in + 10) >> 10 | SafeLoad<uint32_t>(in + 11) << 22, SafeLoad<uint32_t>(in + 11) >> 8 | SafeLoad<uint32_t>(in + 12) << 24, SafeLoad<uint32_t>(in + 12) >> 6 | SafeLoad<uint32_t>(in + 13) << 26, SafeLoad<uint32_t>(in + 13) >> 4 | SafeLoad<uint32_t>(in + 14) << 28, SafeLoad<uint32_t>(in + 14) };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 2 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 30-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 30 | SafeLoad<uint32_t>(in + 16) << 2, SafeLoad<uint32_t>(in + 16) >> 28 | SafeLoad<uint32_t>(in + 17) << 4, SafeLoad<uint32_t>(in + 17) >> 26 | SafeLoad<uint32_t>(in + 18) << 6, SafeLoad<uint32_t>(in + 18) >> 24 | SafeLoad<uint32_t>(in + 19) << 8, SafeLoad<uint32_t>(in + 19) >> 22 | SafeLoad<uint32_t>(in + 20) << 10, SafeLoad<uint32_t>(in + 20) >> 20 | SafeLoad<uint32_t>(in + 21) << 12, SafeLoad<uint32_t>(in + 21) >> 18 | SafeLoad<uint32_t>(in + 22) << 14 };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 30-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 22) >> 16 | SafeLoad<uint32_t>(in + 23) << 16, SafeLoad<uint32_t>(in + 23) >> 14 | SafeLoad<uint32_t>(in + 24) << 18, SafeLoad<uint32_t>(in + 24) >> 12 | SafeLoad<uint32_t>(in + 25) << 20, SafeLoad<uint32_t>(in + 25) >> 10 | SafeLoad<uint32_t>(in + 26) << 22, SafeLoad<uint32_t>(in + 26) >> 8 | SafeLoad<uint32_t>(in + 27) << 24, SafeLoad<uint32_t>(in + 27) >> 6 | SafeLoad<uint32_t>(in + 28) << 26, SafeLoad<uint32_t>(in + 28) >> 4 | SafeLoad<uint32_t>(in + 29) << 28, SafeLoad<uint32_t>(in + 29) };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 2 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 30;
+ return in;
+}
+
+inline static const uint32_t* unpack31_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7fffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 31-bit bundles 0 to 7
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 31 | SafeLoad<uint32_t>(in + 1) << 1, SafeLoad<uint32_t>(in + 1) >> 30 | SafeLoad<uint32_t>(in + 2) << 2, SafeLoad<uint32_t>(in + 2) >> 29 | SafeLoad<uint32_t>(in + 3) << 3, SafeLoad<uint32_t>(in + 3) >> 28 | SafeLoad<uint32_t>(in + 4) << 4, SafeLoad<uint32_t>(in + 4) >> 27 | SafeLoad<uint32_t>(in + 5) << 5, SafeLoad<uint32_t>(in + 5) >> 26 | SafeLoad<uint32_t>(in + 6) << 6, SafeLoad<uint32_t>(in + 6) >> 25 | SafeLoad<uint32_t>(in + 7) << 7 };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 31-bit bundles 8 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7) >> 24 | SafeLoad<uint32_t>(in + 8) << 8, SafeLoad<uint32_t>(in + 8) >> 23 | SafeLoad<uint32_t>(in + 9) << 9, SafeLoad<uint32_t>(in + 9) >> 22 | SafeLoad<uint32_t>(in + 10) << 10, SafeLoad<uint32_t>(in + 10) >> 21 | SafeLoad<uint32_t>(in + 11) << 11, SafeLoad<uint32_t>(in + 11) >> 20 | SafeLoad<uint32_t>(in + 12) << 12, SafeLoad<uint32_t>(in + 12) >> 19 | SafeLoad<uint32_t>(in + 13) << 13, SafeLoad<uint32_t>(in + 13) >> 18 | SafeLoad<uint32_t>(in + 14) << 14, SafeLoad<uint32_t>(in + 14) >> 17 | SafeLoad<uint32_t>(in + 15) << 15 };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 31-bit bundles 16 to 23
+ words = simd_batch{ SafeLoad<uint32_t>(in + 15) >> 16 | SafeLoad<uint32_t>(in + 16) << 16, SafeLoad<uint32_t>(in + 16) >> 15 | SafeLoad<uint32_t>(in + 17) << 17, SafeLoad<uint32_t>(in + 17) >> 14 | SafeLoad<uint32_t>(in + 18) << 18, SafeLoad<uint32_t>(in + 18) >> 13 | SafeLoad<uint32_t>(in + 19) << 19, SafeLoad<uint32_t>(in + 19) >> 12 | SafeLoad<uint32_t>(in + 20) << 20, SafeLoad<uint32_t>(in + 20) >> 11 | SafeLoad<uint32_t>(in + 21) << 21, SafeLoad<uint32_t>(in + 21) >> 10 | SafeLoad<uint32_t>(in + 22) << 22, SafeLoad<uint32_t>(in + 22) >> 9 | SafeLoad<uint32_t>(in + 23) << 23 };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ // extract 31-bit bundles 24 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 23) >> 8 | SafeLoad<uint32_t>(in + 24) << 24, SafeLoad<uint32_t>(in + 24) >> 7 | SafeLoad<uint32_t>(in + 25) << 25, SafeLoad<uint32_t>(in + 25) >> 6 | SafeLoad<uint32_t>(in + 26) << 26, SafeLoad<uint32_t>(in + 26) >> 5 | SafeLoad<uint32_t>(in + 27) << 27, SafeLoad<uint32_t>(in + 27) >> 4 | SafeLoad<uint32_t>(in + 28) << 28, SafeLoad<uint32_t>(in + 28) >> 3 | SafeLoad<uint32_t>(in + 29) << 29, SafeLoad<uint32_t>(in + 29) >> 2 | SafeLoad<uint32_t>(in + 30) << 30, SafeLoad<uint32_t>(in + 30) };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 1 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 8;
+
+ in += 31;
+ return in;
+}
+
+inline static const uint32_t* unpack32_32(const uint32_t* in, uint32_t* out) {
+ memcpy(out, in, 32 * sizeof(*out));
+ in += 32;
+ out += 32;
+
+ return in;
+}
+
+}; // struct UnpackBits256
+
+} // namespace
+} // namespace internal
+} // namespace arrow
+
diff --git a/cpp/src/arrow/util/bpacking_simd512_generated.h b/cpp/src/arrow/util/bpacking_simd512_generated.h
new file mode 100644
index 0000000..2a62c96
--- /dev/null
+++ b/cpp/src/arrow/util/bpacking_simd512_generated.h
@@ -0,0 +1,836 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Automatically generated file; DO NOT EDIT.
+
+#pragma once
+
+#include <cstdint>
+#include <cstring>
+
+#include <xsimd/xsimd.hpp>
+
+#include "arrow/util/dispatch.h"
+#include "arrow/util/ubsan.h"
+
+namespace arrow {
+namespace internal {
+namespace {
+
+using ::arrow::util::SafeLoad;
+
+template <DispatchLevel level>
+struct UnpackBits512 {
+
+using simd_batch = xsimd::batch<uint32_t, 16>;
+
+inline static const uint32_t* unpack0_32(const uint32_t* in, uint32_t* out) {
+ memset(out, 0x0, 32 * sizeof(*out));
+ out += 32;
+
+ return in;
+}
+
+inline static const uint32_t* unpack1_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 1-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 1-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 1;
+ return in;
+}
+
+inline static const uint32_t* unpack2_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 2-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) };
+ shifts = simd_batch{ 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 2-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 2;
+ return in;
+}
+
+inline static const uint32_t* unpack3_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 3-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 0, 1, 4, 7, 10, 13 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 3-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 31 | SafeLoad<uint32_t>(in + 2) << 1, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 16, 19, 22, 25, 28, 0, 2, 5, 8, 11, 14, 17, 20, 23, 26, 29 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 3;
+ return in;
+}
+
+inline static const uint32_t* unpack4_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xf;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 4-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) };
+ shifts = simd_batch{ 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 4-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 4;
+ return in;
+}
+
+inline static const uint32_t* unpack5_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1f;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 5-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 0, 5, 10, 15, 20, 25, 0, 3, 8, 13, 18, 23, 0, 1, 6, 11 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 5-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 31 | SafeLoad<uint32_t>(in + 3) << 1, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 29 | SafeLoad<uint32_t>(in + 4) << 3, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 16, 21, 26, 0, 4, 9, 14, 19, 24, 0, 2, 7, 12, 17, 22, 27 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 5;
+ return in;
+}
+
+inline static const uint32_t* unpack6_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3f;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 6-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) };
+ shifts = simd_batch{ 0, 6, 12, 18, 24, 0, 4, 10, 16, 22, 0, 2, 8, 14, 20, 26 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 6-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 28 | SafeLoad<uint32_t>(in + 5) << 4, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 0, 6, 12, 18, 24, 0, 4, 10, 16, 22, 0, 2, 8, 14, 20, 26 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 6;
+ return in;
+}
+
+inline static const uint32_t* unpack7_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7f;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 7-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 28 | SafeLoad<uint32_t>(in + 1) << 4, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 31 | SafeLoad<uint32_t>(in + 2) << 1, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 27 | SafeLoad<uint32_t>(in + 3) << 5, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 0, 7, 14, 21, 0, 3, 10, 17, 24, 0, 6, 13, 20, 0, 2, 9 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 7-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 26 | SafeLoad<uint32_t>(in + 5) << 6, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 29 | SafeLoad<uint32_t>(in + 6) << 3, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) };
+ shifts = simd_batch{ 16, 23, 0, 5, 12, 19, 0, 1, 8, 15, 22, 0, 4, 11, 18, 25 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 7;
+ return in;
+}
+
+inline static const uint32_t* unpack8_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 8-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) };
+ shifts = simd_batch{ 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 8-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) };
+ shifts = simd_batch{ 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 8;
+ return in;
+}
+
+inline static const uint32_t* unpack9_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1ff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 9-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 27 | SafeLoad<uint32_t>(in + 1) << 5, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 31 | SafeLoad<uint32_t>(in + 2) << 1, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 26 | SafeLoad<uint32_t>(in + 3) << 6, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2, SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 0, 9, 18, 0, 4, 13, 22, 0, 8, 17, 0, 3, 12, 21, 0, 7 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 9-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 25 | SafeLoad<uint32_t>(in + 5) << 7, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 29 | SafeLoad<uint32_t>(in + 6) << 3, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 24 | SafeLoad<uint32_t>(in + 7) << 8, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) };
+ shifts = simd_batch{ 16, 0, 2, 11, 20, 0, 6, 15, 0, 1, 10, 19, 0, 5, 14, 23 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 9;
+ return in;
+}
+
+inline static const uint32_t* unpack10_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3ff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 10-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 26 | SafeLoad<uint32_t>(in + 3) << 6, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) };
+ shifts = simd_batch{ 0, 10, 20, 0, 8, 18, 0, 6, 16, 0, 4, 14, 0, 2, 12, 22 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 10-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 30 | SafeLoad<uint32_t>(in + 6) << 2, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 28 | SafeLoad<uint32_t>(in + 7) << 4, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 26 | SafeLoad<uint32_t>(in + 8) << 6, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 24 | SafeLoad<uint32_t>(in + 9) << 8, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) };
+ shifts = simd_batch{ 0, 10, 20, 0, 8, 18, 0, 6, 16, 0, 4, 14, 0, 2, 12, 22 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 10;
+ return in;
+}
+
+inline static const uint32_t* unpack11_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7ff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 11-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 22 | SafeLoad<uint32_t>(in + 1) << 10, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 23 | SafeLoad<uint32_t>(in + 2) << 9, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 24 | SafeLoad<uint32_t>(in + 3) << 8, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 25 | SafeLoad<uint32_t>(in + 4) << 7, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 26 | SafeLoad<uint32_t>(in + 5) << 6, SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 0, 11, 0, 1, 12, 0, 2, 13, 0, 3, 14, 0, 4, 15, 0, 5 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 11-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 27 | SafeLoad<uint32_t>(in + 6) << 5, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 28 | SafeLoad<uint32_t>(in + 7) << 4, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 29 | SafeLoad<uint32_t>(in + 8) << 3, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 30 | SafeLoad<uint32_t>(in + 9) << 2, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 31 | SafeLoad<uint32_t>(in + 10) << 1, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) };
+ shifts = simd_batch{ 16, 0, 6, 17, 0, 7, 18, 0, 8, 19, 0, 9, 20, 0, 10, 21 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 11;
+ return in;
+}
+
+inline static const uint32_t* unpack12_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xfff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 12-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 24 | SafeLoad<uint32_t>(in + 1) << 8, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 28 | SafeLoad<uint32_t>(in + 5) << 4, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) };
+ shifts = simd_batch{ 0, 12, 0, 4, 16, 0, 8, 20, 0, 12, 0, 4, 16, 0, 8, 20 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 12-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 24 | SafeLoad<uint32_t>(in + 7) << 8, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 24 | SafeLoad<uint32_t>(in + 10) << 8, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 28 | SafeLoad<uint32_t>(in + 11) << 4, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) };
+ shifts = simd_batch{ 0, 12, 0, 4, 16, 0, 8, 20, 0, 12, 0, 4, 16, 0, 8, 20 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 12;
+ return in;
+}
+
+inline static const uint32_t* unpack13_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1fff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 13-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 26 | SafeLoad<uint32_t>(in + 1) << 6, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 20 | SafeLoad<uint32_t>(in + 2) << 12, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 27 | SafeLoad<uint32_t>(in + 3) << 5, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 21 | SafeLoad<uint32_t>(in + 4) << 11, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 28 | SafeLoad<uint32_t>(in + 5) << 4, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 22 | SafeLoad<uint32_t>(in + 6) << 10, SafeLoad<uint32_t>(in + 6) };
+ shifts = simd_batch{ 0, 13, 0, 7, 0, 1, 14, 0, 8, 0, 2, 15, 0, 9, 0, 3 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 13-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 29 | SafeLoad<uint32_t>(in + 7) << 3, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 23 | SafeLoad<uint32_t>(in + 8) << 9, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 30 | SafeLoad<uint32_t>(in + 9) << 2, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 24 | SafeLoad<uint32_t>(in + 10) << 8, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 31 | SafeLoad<uint32_t>(in + 11) << 1, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 25 | SafeLoad<uint32_t>(in + 12) << 7, SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) };
+ shifts = simd_batch{ 16, 0, 10, 0, 4, 17, 0, 11, 0, 5, 18, 0, 12, 0, 6, 19 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 13;
+ return in;
+}
+
+inline static const uint32_t* unpack14_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3fff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 14-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 28 | SafeLoad<uint32_t>(in + 1) << 4, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 24 | SafeLoad<uint32_t>(in + 2) << 8, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 20 | SafeLoad<uint32_t>(in + 3) << 12, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 26 | SafeLoad<uint32_t>(in + 5) << 6, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 22 | SafeLoad<uint32_t>(in + 6) << 10, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) };
+ shifts = simd_batch{ 0, 14, 0, 10, 0, 6, 0, 2, 16, 0, 12, 0, 8, 0, 4, 18 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 14-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 24 | SafeLoad<uint32_t>(in + 9) << 8, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 20 | SafeLoad<uint32_t>(in + 10) << 12, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 30 | SafeLoad<uint32_t>(in + 11) << 2, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 26 | SafeLoad<uint32_t>(in + 12) << 6, SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) >> 22 | SafeLoad<uint32_t>(in + 13) << 10, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) };
+ shifts = simd_batch{ 0, 14, 0, 10, 0, 6, 0, 2, 16, 0, 12, 0, 8, 0, 4, 18 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 14;
+ return in;
+}
+
+inline static const uint32_t* unpack15_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7fff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 15-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 26 | SafeLoad<uint32_t>(in + 3) << 6, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 22 | SafeLoad<uint32_t>(in + 5) << 10, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 20 | SafeLoad<uint32_t>(in + 6) << 12, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 18 | SafeLoad<uint32_t>(in + 7) << 14, SafeLoad<uint32_t>(in + 7) };
+ shifts = simd_batch{ 0, 15, 0, 13, 0, 11, 0, 9, 0, 7, 0, 5, 0, 3, 0, 1 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 15-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 31 | SafeLoad<uint32_t>(in + 8) << 1, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 29 | SafeLoad<uint32_t>(in + 9) << 3, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 27 | SafeLoad<uint32_t>(in + 10) << 5, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 25 | SafeLoad<uint32_t>(in + 11) << 7, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 23 | SafeLoad<uint32_t>(in + 12) << 9, SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) >> 21 | SafeLoad<uint32_t>(in + 13) << 11, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 19 | SafeLoad<uint32_t>(in + 14) << 13, SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) };
+ shifts = simd_batch{ 16, 0, 14, 0, 12, 0, 10, 0, 8, 0, 6, 0, 4, 0, 2, 17 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 15;
+ return in;
+}
+
+inline static const uint32_t* unpack16_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 16-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) };
+ shifts = simd_batch{ 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 16-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) };
+ shifts = simd_batch{ 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 16;
+ return in;
+}
+
+inline static const uint32_t* unpack17_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1ffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 17-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 17 | SafeLoad<uint32_t>(in + 1) << 15, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 19 | SafeLoad<uint32_t>(in + 2) << 13, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 21 | SafeLoad<uint32_t>(in + 3) << 11, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 23 | SafeLoad<uint32_t>(in + 4) << 9, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 25 | SafeLoad<uint32_t>(in + 5) << 7, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 27 | SafeLoad<uint32_t>(in + 6) << 5, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 29 | SafeLoad<uint32_t>(in + 7) << 3, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 31 | SafeLoad<uint32_t>(in + 8) << 1 };
+ shifts = simd_batch{ 0, 0, 2, 0, 4, 0, 6, 0, 8, 0, 10, 0, 12, 0, 14, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 17-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 8) >> 16 | SafeLoad<uint32_t>(in + 9) << 16, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 18 | SafeLoad<uint32_t>(in + 10) << 14, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 20 | SafeLoad<uint32_t>(in + 11) << 12, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 22 | SafeLoad<uint32_t>(in + 12) << 10, SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) >> 24 | SafeLoad<uint32_t>(in + 13) << 8, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 26 | SafeLoad<uint32_t>(in + 14) << 6, SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) >> 28 | SafeLoad<uint32_t>(in + 15) << 4, SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 30 | SafeLoad<uint32_t>(in + 16) << 2, SafeLoad<uint32_t>(in + 16) };
+ shifts = simd_batch{ 0, 1, 0, 3, 0, 5, 0, 7, 0, 9, 0, 11, 0, 13, 0, 15 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 17;
+ return in;
+}
+
+inline static const uint32_t* unpack18_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3ffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 18-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 18 | SafeLoad<uint32_t>(in + 1) << 14, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 22 | SafeLoad<uint32_t>(in + 2) << 10, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 26 | SafeLoad<uint32_t>(in + 3) << 6, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2, SafeLoad<uint32_t>(in + 4) >> 16 | SafeLoad<uint32_t>(in + 5) << 16, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 20 | SafeLoad<uint32_t>(in + 6) << 12, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 24 | SafeLoad<uint32_t>(in + 7) << 8, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8) };
+ shifts = simd_batch{ 0, 0, 4, 0, 8, 0, 12, 0, 0, 2, 0, 6, 0, 10, 0, 14 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 18-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 18 | SafeLoad<uint32_t>(in + 10) << 14, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 22 | SafeLoad<uint32_t>(in + 11) << 10, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 26 | SafeLoad<uint32_t>(in + 12) << 6, SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) >> 30 | SafeLoad<uint32_t>(in + 13) << 2, SafeLoad<uint32_t>(in + 13) >> 16 | SafeLoad<uint32_t>(in + 14) << 16, SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) >> 20 | SafeLoad<uint32_t>(in + 15) << 12, SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 24 | SafeLoad<uint32_t>(in + 16) << 8, SafeLoad<uint32_t>(in + 16), SafeLoad<uint32_t>(in + 16) >> 28 | SafeLoad<uint32_t>(in + 17) << 4, SafeLoad<uint32_t>(in + 17) };
+ shifts = simd_batch{ 0, 0, 4, 0, 8, 0, 12, 0, 0, 2, 0, 6, 0, 10, 0, 14 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 18;
+ return in;
+}
+
+inline static const uint32_t* unpack19_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7ffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 19-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 19 | SafeLoad<uint32_t>(in + 1) << 13, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 25 | SafeLoad<uint32_t>(in + 2) << 7, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 31 | SafeLoad<uint32_t>(in + 3) << 1, SafeLoad<uint32_t>(in + 3) >> 18 | SafeLoad<uint32_t>(in + 4) << 14, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 24 | SafeLoad<uint32_t>(in + 5) << 8, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 30 | SafeLoad<uint32_t>(in + 6) << 2, SafeLoad<uint32_t>(in + 6) >> 17 | SafeLoad<uint32_t>(in + 7) << 15, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 23 | SafeLoad<uint32_t>(in + 8) << 9, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 29 | SafeLoad<uint32_t>(in + 9) << 3 };
+ shifts = simd_batch{ 0, 0, 6, 0, 12, 0, 0, 5, 0, 11, 0, 0, 4, 0, 10, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 19-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 9) >> 16 | SafeLoad<uint32_t>(in + 10) << 16, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 22 | SafeLoad<uint32_t>(in + 11) << 10, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 28 | SafeLoad<uint32_t>(in + 12) << 4, SafeLoad<uint32_t>(in + 12) >> 15 | SafeLoad<uint32_t>(in + 13) << 17, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 21 | SafeLoad<uint32_t>(in + 14) << 11, SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) >> 27 | SafeLoad<uint32_t>(in + 15) << 5, SafeLoad<uint32_t>(in + 15) >> 14 | SafeLoad<uint32_t>(in + 16) << 18, SafeLoad<uint32_t>(in + 16), SafeLoad<uint32_t>(in + 16) >> 20 | SafeLoad<uint32_t>(in + 17) << 12, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 26 | SafeLoad<uint32_t>(in + 18) << 6, SafeLoad<uint32_t>(in + 18) };
+ shifts = simd_batch{ 0, 3, 0, 9, 0, 0, 2, 0, 8, 0, 0, 1, 0, 7, 0, 13 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 19;
+ return in;
+}
+
+inline static const uint32_t* unpack20_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xfffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 20-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 20 | SafeLoad<uint32_t>(in + 1) << 12, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2) >> 16 | SafeLoad<uint32_t>(in + 3) << 16, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 20 | SafeLoad<uint32_t>(in + 6) << 12, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 28 | SafeLoad<uint32_t>(in + 7) << 4, SafeLoad<uint32_t>(in + 7) >> 16 | SafeLoad<uint32_t>(in + 8) << 16, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 24 | SafeLoad<uint32_t>(in + 9) << 8, SafeLoad<uint32_t>(in + 9) };
+ shifts = simd_batch{ 0, 0, 8, 0, 0, 4, 0, 12, 0, 0, 8, 0, 0, 4, 0, 12 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 20-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 20 | SafeLoad<uint32_t>(in + 11) << 12, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 28 | SafeLoad<uint32_t>(in + 12) << 4, SafeLoad<uint32_t>(in + 12) >> 16 | SafeLoad<uint32_t>(in + 13) << 16, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 24 | SafeLoad<uint32_t>(in + 14) << 8, SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 20 | SafeLoad<uint32_t>(in + 16) << 12, SafeLoad<uint32_t>(in + 16), SafeLoad<uint32_t>(in + 16) >> 28 | SafeLoad<uint32_t>(in + 17) << 4, SafeLoad<uint32_t>(in + 17) >> 16 | SafeLoad<uint32_t>(in + 18) << 16, SafeLoad<uint32_t>(in + 18), SafeLoad<uint32_t>(in + 18) >> 24 | SafeLoad<uint32_t>(in + 19) << 8, SafeLoad<uint32_t>(in + 19) };
+ shifts = simd_batch{ 0, 0, 8, 0, 0, 4, 0, 12, 0, 0, 8, 0, 0, 4, 0, 12 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 20;
+ return in;
+}
+
+inline static const uint32_t* unpack21_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1fffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 21-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 21 | SafeLoad<uint32_t>(in + 1) << 11, SafeLoad<uint32_t>(in + 1), SafeLoad<uint32_t>(in + 1) >> 31 | SafeLoad<uint32_t>(in + 2) << 1, SafeLoad<uint32_t>(in + 2) >> 20 | SafeLoad<uint32_t>(in + 3) << 12, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 30 | SafeLoad<uint32_t>(in + 4) << 2, SafeLoad<uint32_t>(in + 4) >> 19 | SafeLoad<uint32_t>(in + 5) << 13, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 29 | SafeLoad<uint32_t>(in + 6) << 3, SafeLoad<uint32_t>(in + 6) >> 18 | SafeLoad<uint32_t>(in + 7) << 14, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8) >> 17 | SafeLoad<uint32_t>(in + 9) << 15, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 27 | SafeLoad<uint32_t>(in + 10) << 5 };
+ shifts = simd_batch{ 0, 0, 10, 0, 0, 9, 0, 0, 8, 0, 0, 7, 0, 0, 6, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 21-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 10) >> 16 | SafeLoad<uint32_t>(in + 11) << 16, SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 26 | SafeLoad<uint32_t>(in + 12) << 6, SafeLoad<uint32_t>(in + 12) >> 15 | SafeLoad<uint32_t>(in + 13) << 17, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 25 | SafeLoad<uint32_t>(in + 14) << 7, SafeLoad<uint32_t>(in + 14) >> 14 | SafeLoad<uint32_t>(in + 15) << 18, SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 24 | SafeLoad<uint32_t>(in + 16) << 8, SafeLoad<uint32_t>(in + 16) >> 13 | SafeLoad<uint32_t>(in + 17) << 19, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 23 | SafeLoad<uint32_t>(in + 18) << 9, SafeLoad<uint32_t>(in + 18) >> 12 | SafeLoad<uint32_t>(in + 19) << 20, SafeLoad<uint32_t>(in + 19), SafeLoad<uint32_t>(in + 19) >> 22 | SafeLoad<uint32_t>(in + 20) << 10, SafeLoad<uint32_t>(in + 20) };
+ shifts = simd_batch{ 0, 5, 0, 0, 4, 0, 0, 3, 0, 0, 2, 0, 0, 1, 0, 11 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 21;
+ return in;
+}
+
+inline static const uint32_t* unpack22_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3fffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 22-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 22 | SafeLoad<uint32_t>(in + 1) << 10, SafeLoad<uint32_t>(in + 1) >> 12 | SafeLoad<uint32_t>(in + 2) << 20, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 24 | SafeLoad<uint32_t>(in + 3) << 8, SafeLoad<uint32_t>(in + 3) >> 14 | SafeLoad<uint32_t>(in + 4) << 18, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 26 | SafeLoad<uint32_t>(in + 5) << 6, SafeLoad<uint32_t>(in + 5) >> 16 | SafeLoad<uint32_t>(in + 6) << 16, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 28 | SafeLoad<uint32_t>(in + 7) << 4, SafeLoad<uint32_t>(in + 7) >> 18 | SafeLoad<uint32_t>(in + 8) << 14, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 30 | SafeLoad<uint32_t>(in + 9) << 2, SafeLoad<uint32_t>(in + 9) >> 20 | SafeLoad<uint32_t>(in + 10) << 12, SafeLoad<uint32_t>(in + 10) };
+ shifts = simd_batch{ 0, 0, 0, 2, 0, 0, 4, 0, 0, 6, 0, 0, 8, 0, 0, 10 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 22-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 11), SafeLoad<uint32_t>(in + 11) >> 22 | SafeLoad<uint32_t>(in + 12) << 10, SafeLoad<uint32_t>(in + 12) >> 12 | SafeLoad<uint32_t>(in + 13) << 20, SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 24 | SafeLoad<uint32_t>(in + 14) << 8, SafeLoad<uint32_t>(in + 14) >> 14 | SafeLoad<uint32_t>(in + 15) << 18, SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 26 | SafeLoad<uint32_t>(in + 16) << 6, SafeLoad<uint32_t>(in + 16) >> 16 | SafeLoad<uint32_t>(in + 17) << 16, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 28 | SafeLoad<uint32_t>(in + 18) << 4, SafeLoad<uint32_t>(in + 18) >> 18 | SafeLoad<uint32_t>(in + 19) << 14, SafeLoad<uint32_t>(in + 19), SafeLoad<uint32_t>(in + 19) >> 30 | SafeLoad<uint32_t>(in + 20) << 2, SafeLoad<uint32_t>(in + 20) >> 20 | SafeLoad<uint32_t>(in + 21) << 12, SafeLoad<uint32_t>(in + 21) };
+ shifts = simd_batch{ 0, 0, 0, 2, 0, 0, 4, 0, 0, 6, 0, 0, 8, 0, 0, 10 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 22;
+ return in;
+}
+
+inline static const uint32_t* unpack23_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7fffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 23-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 23 | SafeLoad<uint32_t>(in + 1) << 9, SafeLoad<uint32_t>(in + 1) >> 14 | SafeLoad<uint32_t>(in + 2) << 18, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 2) >> 28 | SafeLoad<uint32_t>(in + 3) << 4, SafeLoad<uint32_t>(in + 3) >> 19 | SafeLoad<uint32_t>(in + 4) << 13, SafeLoad<uint32_t>(in + 4) >> 10 | SafeLoad<uint32_t>(in + 5) << 22, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 24 | SafeLoad<uint32_t>(in + 6) << 8, SafeLoad<uint32_t>(in + 6) >> 15 | SafeLoad<uint32_t>(in + 7) << 17, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 29 | SafeLoad<uint32_t>(in + 8) << 3, SafeLoad<uint32_t>(in + 8) >> 20 | SafeLoad<uint32_t>(in + 9) << 12, SafeLoad<uint32_t>(in + 9) >> 11 | SafeLoad<uint32_t>(in + 10) << 21, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 25 | SafeLoad<uint32_t>(in + 11) << 7 };
+ shifts = simd_batch{ 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 2, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 23-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 11) >> 16 | SafeLoad<uint32_t>(in + 12) << 16, SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) >> 30 | SafeLoad<uint32_t>(in + 13) << 2, SafeLoad<uint32_t>(in + 13) >> 21 | SafeLoad<uint32_t>(in + 14) << 11, SafeLoad<uint32_t>(in + 14) >> 12 | SafeLoad<uint32_t>(in + 15) << 20, SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 26 | SafeLoad<uint32_t>(in + 16) << 6, SafeLoad<uint32_t>(in + 16) >> 17 | SafeLoad<uint32_t>(in + 17) << 15, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 31 | SafeLoad<uint32_t>(in + 18) << 1, SafeLoad<uint32_t>(in + 18) >> 22 | SafeLoad<uint32_t>(in + 19) << 10, SafeLoad<uint32_t>(in + 19) >> 13 | SafeLoad<uint32_t>(in + 20) << 19, SafeLoad<uint32_t>(in + 20), SafeLoad<uint32_t>(in + 20) >> 27 | SafeLoad<uint32_t>(in + 21) << 5, SafeLoad<uint32_t>(in + 21) >> 18 | SafeLoad<uint32_t>(in + 22) << 14, SafeLoad<uint32_t>(in + 22) };
+ shifts = simd_batch{ 0, 7, 0, 0, 0, 3, 0, 0, 8, 0, 0, 0, 4, 0, 0, 9 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 23;
+ return in;
+}
+
+inline static const uint32_t* unpack24_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 24-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 24 | SafeLoad<uint32_t>(in + 1) << 8, SafeLoad<uint32_t>(in + 1) >> 16 | SafeLoad<uint32_t>(in + 2) << 16, SafeLoad<uint32_t>(in + 2), SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4) >> 16 | SafeLoad<uint32_t>(in + 5) << 16, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 6) >> 24 | SafeLoad<uint32_t>(in + 7) << 8, SafeLoad<uint32_t>(in + 7) >> 16 | SafeLoad<uint32_t>(in + 8) << 16, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 24 | SafeLoad<uint32_t>(in + 10) << 8, SafeLoad<uint32_t>(in + 10) >> 16 | SafeLoad<uint32_t>(in + 11) << 16, SafeLoad<uint32_t>(in + 11) };
+ shifts = simd_batch{ 0, 0, 0, 8, 0, 0, 0, 8, 0, 0, 0, 8, 0, 0, 0, 8 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 24-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 12), SafeLoad<uint32_t>(in + 12) >> 24 | SafeLoad<uint32_t>(in + 13) << 8, SafeLoad<uint32_t>(in + 13) >> 16 | SafeLoad<uint32_t>(in + 14) << 16, SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 24 | SafeLoad<uint32_t>(in + 16) << 8, SafeLoad<uint32_t>(in + 16) >> 16 | SafeLoad<uint32_t>(in + 17) << 16, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 18), SafeLoad<uint32_t>(in + 18) >> 24 | SafeLoad<uint32_t>(in + 19) << 8, SafeLoad<uint32_t>(in + 19) >> 16 | SafeLoad<uint32_t>(in + 20) << 16, SafeLoad<uint32_t>(in + 20), SafeLoad<uint32_t>(in + 21), SafeLoad<uint32_t>(in + 21) >> 24 | SafeLoad<uint32_t>(in + 22) << 8, SafeLoad<uint32_t>(in + 22) >> 16 | SafeLoad<uint32_t>(in + 23) << 16, SafeLoad<uint32_t>(in + 23) };
+ shifts = simd_batch{ 0, 0, 0, 8, 0, 0, 0, 8, 0, 0, 0, 8, 0, 0, 0, 8 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 24;
+ return in;
+}
+
+inline static const uint32_t* unpack25_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1ffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 25-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 25 | SafeLoad<uint32_t>(in + 1) << 7, SafeLoad<uint32_t>(in + 1) >> 18 | SafeLoad<uint32_t>(in + 2) << 14, SafeLoad<uint32_t>(in + 2) >> 11 | SafeLoad<uint32_t>(in + 3) << 21, SafeLoad<uint32_t>(in + 3), SafeLoad<uint32_t>(in + 3) >> 29 | SafeLoad<uint32_t>(in + 4) << 3, SafeLoad<uint32_t>(in + 4) >> 22 | SafeLoad<uint32_t>(in + 5) << 10, SafeLoad<uint32_t>(in + 5) >> 15 | SafeLoad<uint32_t>(in + 6) << 17, SafeLoad<uint32_t>(in + 6) >> 8 | SafeLoad<uint32_t>(in + 7) << 24, SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 26 | SafeLoad<uint32_t>(in + 8) << 6, SafeLoad<uint32_t>(in + 8) >> 19 | SafeLoad<uint32_t>(in + 9) << 13, SafeLoad<uint32_t>(in + 9) >> 12 | SafeLoad<uint32_t>(in + 10) << 20, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 30 | SafeLoad<uint32_t>(in + 11) << 2, SafeLoad<uint32_t>(in + 11) >> 23 | SafeLoad<uint32_t>(in + 12) << 9 };
+ shifts = simd_batch{ 0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 25-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 12) >> 16 | SafeLoad<uint32_t>(in + 13) << 16, SafeLoad<uint32_t>(in + 13) >> 9 | SafeLoad<uint32_t>(in + 14) << 23, SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) >> 27 | SafeLoad<uint32_t>(in + 15) << 5, SafeLoad<uint32_t>(in + 15) >> 20 | SafeLoad<uint32_t>(in + 16) << 12, SafeLoad<uint32_t>(in + 16) >> 13 | SafeLoad<uint32_t>(in + 17) << 19, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 31 | SafeLoad<uint32_t>(in + 18) << 1, SafeLoad<uint32_t>(in + 18) >> 24 | SafeLoad<uint32_t>(in + 19) << 8, SafeLoad<uint32_t>(in + 19) >> 17 | SafeLoad<uint32_t>(in + 20) << 15, SafeLoad<uint32_t>(in + 20) >> 10 | SafeLoad<uint32_t>(in + 21) << 22, SafeLoad<uint32_t>(in + 21), SafeLoad<uint32_t>(in + 21) >> 28 | SafeLoad<uint32_t>(in + 22) << 4, SafeLoad<uint32_t>(in + 22) >> 21 | SafeLoad<uint32_t>(in + 23) << 11, SafeLoad<uint32_t>(in + 23) >> 14 | SafeLoad<uint32_t>(in + 24) << 18, SafeLoad<uint32_t>(in + 24) };
+ shifts = simd_batch{ 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 0, 3, 0, 0, 0, 7 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 25;
+ return in;
+}
+
+inline static const uint32_t* unpack26_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3ffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 26-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 26 | SafeLoad<uint32_t>(in + 1) << 6, SafeLoad<uint32_t>(in + 1) >> 20 | SafeLoad<uint32_t>(in + 2) << 12, SafeLoad<uint32_t>(in + 2) >> 14 | SafeLoad<uint32_t>(in + 3) << 18, SafeLoad<uint32_t>(in + 3) >> 8 | SafeLoad<uint32_t>(in + 4) << 24, SafeLoad<uint32_t>(in + 4), SafeLoad<uint32_t>(in + 4) >> 28 | SafeLoad<uint32_t>(in + 5) << 4, SafeLoad<uint32_t>(in + 5) >> 22 | SafeLoad<uint32_t>(in + 6) << 10, SafeLoad<uint32_t>(in + 6) >> 16 | SafeLoad<uint32_t>(in + 7) << 16, SafeLoad<uint32_t>(in + 7) >> 10 | SafeLoad<uint32_t>(in + 8) << 22, SafeLoad<uint32_t>(in + 8), SafeLoad<uint32_t>(in + 8) >> 30 | SafeLoad<uint32_t>(in + 9) << 2, SafeLoad<uint32_t>(in + 9) >> 24 | SafeLoad<uint32_t>(in + 10) << 8, SafeLoad<uint32_t>(in + 10) >> 18 | SafeLoad<uint32_t>(in + 11) << 14, SafeLoad<uint32_t>(in + 11) >> 12 | SafeLoad<uint32_t>(in + 12) << 20, SafeLoad<uint32_t>(in + 12) };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4, 0, 0, 0, 0, 6 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 26-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 13), SafeLoad<uint32_t>(in + 13) >> 26 | SafeLoad<uint32_t>(in + 14) << 6, SafeLoad<uint32_t>(in + 14) >> 20 | SafeLoad<uint32_t>(in + 15) << 12, SafeLoad<uint32_t>(in + 15) >> 14 | SafeLoad<uint32_t>(in + 16) << 18, SafeLoad<uint32_t>(in + 16) >> 8 | SafeLoad<uint32_t>(in + 17) << 24, SafeLoad<uint32_t>(in + 17), SafeLoad<uint32_t>(in + 17) >> 28 | SafeLoad<uint32_t>(in + 18) << 4, SafeLoad<uint32_t>(in + 18) >> 22 | SafeLoad<uint32_t>(in + 19) << 10, SafeLoad<uint32_t>(in + 19) >> 16 | SafeLoad<uint32_t>(in + 20) << 16, SafeLoad<uint32_t>(in + 20) >> 10 | SafeLoad<uint32_t>(in + 21) << 22, SafeLoad<uint32_t>(in + 21), SafeLoad<uint32_t>(in + 21) >> 30 | SafeLoad<uint32_t>(in + 22) << 2, SafeLoad<uint32_t>(in + 22) >> 24 | SafeLoad<uint32_t>(in + 23) << 8, SafeLoad<uint32_t>(in + 23) >> 18 | SafeLoad<uint32_t>(in + 24) << 14, SafeLoad<uint32_t>(in + 24) >> 12 | SafeLoad<uint32_t>(in + 25) << 20, SafeLoad<uint32_t>(in + 25) };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4, 0, 0, 0, 0, 6 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 26;
+ return in;
+}
+
+inline static const uint32_t* unpack27_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7ffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 27-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 27 | SafeLoad<uint32_t>(in + 1) << 5, SafeLoad<uint32_t>(in + 1) >> 22 | SafeLoad<uint32_t>(in + 2) << 10, SafeLoad<uint32_t>(in + 2) >> 17 | SafeLoad<uint32_t>(in + 3) << 15, SafeLoad<uint32_t>(in + 3) >> 12 | SafeLoad<uint32_t>(in + 4) << 20, SafeLoad<uint32_t>(in + 4) >> 7 | SafeLoad<uint32_t>(in + 5) << 25, SafeLoad<uint32_t>(in + 5), SafeLoad<uint32_t>(in + 5) >> 29 | SafeLoad<uint32_t>(in + 6) << 3, SafeLoad<uint32_t>(in + 6) >> 24 | SafeLoad<uint32_t>(in + 7) << 8, SafeLoad<uint32_t>(in + 7) >> 19 | SafeLoad<uint32_t>(in + 8) << 13, SafeLoad<uint32_t>(in + 8) >> 14 | SafeLoad<uint32_t>(in + 9) << 18, SafeLoad<uint32_t>(in + 9) >> 9 | SafeLoad<uint32_t>(in + 10) << 23, SafeLoad<uint32_t>(in + 10), SafeLoad<uint32_t>(in + 10) >> 31 | SafeLoad<uint32_t>(in + 11) << 1, SafeLoad<uint32_t>(in + 11) >> 26 | SafeLoad<uint32_t>(in + 12) << 6, SafeLoad<uint32_t>(in + 12) >> 21 | SafeLoad<uint32_t>(in + 13) << 11 };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 27-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 13) >> 16 | SafeLoad<uint32_t>(in + 14) << 16, SafeLoad<uint32_t>(in + 14) >> 11 | SafeLoad<uint32_t>(in + 15) << 21, SafeLoad<uint32_t>(in + 15) >> 6 | SafeLoad<uint32_t>(in + 16) << 26, SafeLoad<uint32_t>(in + 16), SafeLoad<uint32_t>(in + 16) >> 28 | SafeLoad<uint32_t>(in + 17) << 4, SafeLoad<uint32_t>(in + 17) >> 23 | SafeLoad<uint32_t>(in + 18) << 9, SafeLoad<uint32_t>(in + 18) >> 18 | SafeLoad<uint32_t>(in + 19) << 14, SafeLoad<uint32_t>(in + 19) >> 13 | SafeLoad<uint32_t>(in + 20) << 19, SafeLoad<uint32_t>(in + 20) >> 8 | SafeLoad<uint32_t>(in + 21) << 24, SafeLoad<uint32_t>(in + 21), SafeLoad<uint32_t>(in + 21) >> 30 | SafeLoad<uint32_t>(in + 22) << 2, SafeLoad<uint32_t>(in + 22) >> 25 | SafeLoad<uint32_t>(in + 23) << 7, SafeLoad<uint32_t>(in + 23) >> 20 | SafeLoad<uint32_t>(in + 24) << 12, SafeLoad<uint32_t>(in + 24) >> 15 | SafeLoad<uint32_t>(in + 25) << 17, SafeLoad<uint32_t>(in + 25) >> 10 | SafeLoad<uint32_t>(in + 26) << 22, SafeLoad<uint32_t>(in + 26) };
+ shifts = simd_batch{ 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 5 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 27;
+ return in;
+}
+
+inline static const uint32_t* unpack28_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0xfffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 28-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 28 | SafeLoad<uint32_t>(in + 1) << 4, SafeLoad<uint32_t>(in + 1) >> 24 | SafeLoad<uint32_t>(in + 2) << 8, SafeLoad<uint32_t>(in + 2) >> 20 | SafeLoad<uint32_t>(in + 3) << 12, SafeLoad<uint32_t>(in + 3) >> 16 | SafeLoad<uint32_t>(in + 4) << 16, SafeLoad<uint32_t>(in + 4) >> 12 | SafeLoad<uint32_t>(in + 5) << 20, SafeLoad<uint32_t>(in + 5) >> 8 | SafeLoad<uint32_t>(in + 6) << 24, SafeLoad<uint32_t>(in + 6), SafeLoad<uint32_t>(in + 7), SafeLoad<uint32_t>(in + 7) >> 28 | SafeLoad<uint32_t>(in + 8) << 4, SafeLoad<uint32_t>(in + 8) >> 24 | SafeLoad<uint32_t>(in + 9) << 8, SafeLoad<uint32_t>(in + 9) >> 20 | SafeLoad<uint32_t>(in + 10) << 12, SafeLoad<uint32_t>(in + 10) >> 16 | SafeLoad<uint32_t>(in + 11) << 16, SafeLoad<uint32_t>(in + 11) >> 12 | SafeLoad<uint32_t>(in + 12) << 20, SafeLoad<uint32_t>(in + 12) >> 8 | SafeLoad<uint32_t>(in + 13) << 24, SafeLoad<uint32_t>(in + 13) };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 28-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 14), SafeLoad<uint32_t>(in + 14) >> 28 | SafeLoad<uint32_t>(in + 15) << 4, SafeLoad<uint32_t>(in + 15) >> 24 | SafeLoad<uint32_t>(in + 16) << 8, SafeLoad<uint32_t>(in + 16) >> 20 | SafeLoad<uint32_t>(in + 17) << 12, SafeLoad<uint32_t>(in + 17) >> 16 | SafeLoad<uint32_t>(in + 18) << 16, SafeLoad<uint32_t>(in + 18) >> 12 | SafeLoad<uint32_t>(in + 19) << 20, SafeLoad<uint32_t>(in + 19) >> 8 | SafeLoad<uint32_t>(in + 20) << 24, SafeLoad<uint32_t>(in + 20), SafeLoad<uint32_t>(in + 21), SafeLoad<uint32_t>(in + 21) >> 28 | SafeLoad<uint32_t>(in + 22) << 4, SafeLoad<uint32_t>(in + 22) >> 24 | SafeLoad<uint32_t>(in + 23) << 8, SafeLoad<uint32_t>(in + 23) >> 20 | SafeLoad<uint32_t>(in + 24) << 12, SafeLoad<uint32_t>(in + 24) >> 16 | SafeLoad<uint32_t>(in + 25) << 16, SafeLoad<uint32_t>(in + 25) >> 12 | SafeLoad<uint32_t>(in + 26) << 20, SafeLoad<uint32_t>(in + 26) >> 8 | SafeLoad<uint32_t>(in + 27) << 24, SafeLoad<uint32_t>(in + 27) };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 28;
+ return in;
+}
+
+inline static const uint32_t* unpack29_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x1fffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 29-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 29 | SafeLoad<uint32_t>(in + 1) << 3, SafeLoad<uint32_t>(in + 1) >> 26 | SafeLoad<uint32_t>(in + 2) << 6, SafeLoad<uint32_t>(in + 2) >> 23 | SafeLoad<uint32_t>(in + 3) << 9, SafeLoad<uint32_t>(in + 3) >> 20 | SafeLoad<uint32_t>(in + 4) << 12, SafeLoad<uint32_t>(in + 4) >> 17 | SafeLoad<uint32_t>(in + 5) << 15, SafeLoad<uint32_t>(in + 5) >> 14 | SafeLoad<uint32_t>(in + 6) << 18, SafeLoad<uint32_t>(in + 6) >> 11 | SafeLoad<uint32_t>(in + 7) << 21, SafeLoad<uint32_t>(in + 7) >> 8 | SafeLoad<uint32_t>(in + 8) << 24, SafeLoad<uint32_t>(in + 8) >> 5 | SafeLoad<uint32_t>(in + 9) << 27, SafeLoad<uint32_t>(in + 9), SafeLoad<uint32_t>(in + 9) >> 31 | SafeLoad<uint32_t>(in + 10) << 1, SafeLoad<uint32_t>(in + 10) >> 28 | SafeLoad<uint32_t>(in + 11) << 4, SafeLoad<uint32_t>(in + 11) >> 25 | SafeLoad<uint32_t>(in + 12) << 7, SafeLoad<uint32_t>(in + 12) >> 22 | SafeLoad<uint32_t>(in + 13) << 10, SafeLoad<uint32_t>(in + 13) >> 19 | SafeLoad<uint32_t>(in + 14) << 13 };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 29-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 14) >> 16 | SafeLoad<uint32_t>(in + 15) << 16, SafeLoad<uint32_t>(in + 15) >> 13 | SafeLoad<uint32_t>(in + 16) << 19, SafeLoad<uint32_t>(in + 16) >> 10 | SafeLoad<uint32_t>(in + 17) << 22, SafeLoad<uint32_t>(in + 17) >> 7 | SafeLoad<uint32_t>(in + 18) << 25, SafeLoad<uint32_t>(in + 18) >> 4 | SafeLoad<uint32_t>(in + 19) << 28, SafeLoad<uint32_t>(in + 19), SafeLoad<uint32_t>(in + 19) >> 30 | SafeLoad<uint32_t>(in + 20) << 2, SafeLoad<uint32_t>(in + 20) >> 27 | SafeLoad<uint32_t>(in + 21) << 5, SafeLoad<uint32_t>(in + 21) >> 24 | SafeLoad<uint32_t>(in + 22) << 8, SafeLoad<uint32_t>(in + 22) >> 21 | SafeLoad<uint32_t>(in + 23) << 11, SafeLoad<uint32_t>(in + 23) >> 18 | SafeLoad<uint32_t>(in + 24) << 14, SafeLoad<uint32_t>(in + 24) >> 15 | SafeLoad<uint32_t>(in + 25) << 17, SafeLoad<uint32_t>(in + 25) >> 12 | SafeLoad<uint32_t>(in + 26) << 20, SafeLoad<uint32_t>(in + 26) >> 9 | SafeLoad<uint32_t>(in + 27) << 23, SafeLoad<uint32_t>(in + 27) >> 6 | SafeLoad<uint32_t>(in + 28) << 26, SafeLoad<uint32_t>(in + 28) };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 29;
+ return in;
+}
+
+inline static const uint32_t* unpack30_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x3fffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 30-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 30 | SafeLoad<uint32_t>(in + 1) << 2, SafeLoad<uint32_t>(in + 1) >> 28 | SafeLoad<uint32_t>(in + 2) << 4, SafeLoad<uint32_t>(in + 2) >> 26 | SafeLoad<uint32_t>(in + 3) << 6, SafeLoad<uint32_t>(in + 3) >> 24 | SafeLoad<uint32_t>(in + 4) << 8, SafeLoad<uint32_t>(in + 4) >> 22 | SafeLoad<uint32_t>(in + 5) << 10, SafeLoad<uint32_t>(in + 5) >> 20 | SafeLoad<uint32_t>(in + 6) << 12, SafeLoad<uint32_t>(in + 6) >> 18 | SafeLoad<uint32_t>(in + 7) << 14, SafeLoad<uint32_t>(in + 7) >> 16 | SafeLoad<uint32_t>(in + 8) << 16, SafeLoad<uint32_t>(in + 8) >> 14 | SafeLoad<uint32_t>(in + 9) << 18, SafeLoad<uint32_t>(in + 9) >> 12 | SafeLoad<uint32_t>(in + 10) << 20, SafeLoad<uint32_t>(in + 10) >> 10 | SafeLoad<uint32_t>(in + 11) << 22, SafeLoad<uint32_t>(in + 11) >> 8 | SafeLoad<uint32_t>(in + 12) << 24, SafeLoad<uint32_t>(in + 12) >> 6 | SafeLoad<uint32_t>(in + 13) << 26, SafeLoad<uint32_t>(in + 13) >> 4 | SafeLoad<uint32_t>(in + 14) << 28, SafeLoad<uint32_t>(in + 14) };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 30-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 15), SafeLoad<uint32_t>(in + 15) >> 30 | SafeLoad<uint32_t>(in + 16) << 2, SafeLoad<uint32_t>(in + 16) >> 28 | SafeLoad<uint32_t>(in + 17) << 4, SafeLoad<uint32_t>(in + 17) >> 26 | SafeLoad<uint32_t>(in + 18) << 6, SafeLoad<uint32_t>(in + 18) >> 24 | SafeLoad<uint32_t>(in + 19) << 8, SafeLoad<uint32_t>(in + 19) >> 22 | SafeLoad<uint32_t>(in + 20) << 10, SafeLoad<uint32_t>(in + 20) >> 20 | SafeLoad<uint32_t>(in + 21) << 12, SafeLoad<uint32_t>(in + 21) >> 18 | SafeLoad<uint32_t>(in + 22) << 14, SafeLoad<uint32_t>(in + 22) >> 16 | SafeLoad<uint32_t>(in + 23) << 16, SafeLoad<uint32_t>(in + 23) >> 14 | SafeLoad<uint32_t>(in + 24) << 18, SafeLoad<uint32_t>(in + 24) >> 12 | SafeLoad<uint32_t>(in + 25) << 20, SafeLoad<uint32_t>(in + 25) >> 10 | SafeLoad<uint32_t>(in + 26) << 22, SafeLoad<uint32_t>(in + 26) >> 8 | SafeLoad<uint32_t>(in + 27) << 24, SafeLoad<uint32_t>(in + 27) >> 6 | SafeLoad<uint32_t>(in + 28) << 26, SafeLoad<uint32_t>(in + 28) >> 4 | SafeLoad<uint32_t>(in + 29) << 28, SafeLoad<uint32_t>(in + 29) };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 30;
+ return in;
+}
+
+inline static const uint32_t* unpack31_32(const uint32_t* in, uint32_t* out) {
+ uint32_t mask = 0x7fffffff;
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+
+ // extract 31-bit bundles 0 to 15
+ words = simd_batch{ SafeLoad<uint32_t>(in + 0), SafeLoad<uint32_t>(in + 0) >> 31 | SafeLoad<uint32_t>(in + 1) << 1, SafeLoad<uint32_t>(in + 1) >> 30 | SafeLoad<uint32_t>(in + 2) << 2, SafeLoad<uint32_t>(in + 2) >> 29 | SafeLoad<uint32_t>(in + 3) << 3, SafeLoad<uint32_t>(in + 3) >> 28 | SafeLoad<uint32_t>(in + 4) << 4, SafeLoad<uint32_t>(in + 4) >> 27 | SafeLoad<uint32_t>(in + 5) << 5, SafeLoad<uint32_t>(in + 5) >> 26 | SafeLoad<uint32_t>(in + 6) << 6, SafeLoad<uint32_t>(in + 6) >> 25 | SafeLoad<uint32_t>(in + 7) << 7, SafeLoad<uint32_t>(in + 7) >> 24 | SafeLoad<uint32_t>(in + 8) << 8, SafeLoad<uint32_t>(in + 8) >> 23 | SafeLoad<uint32_t>(in + 9) << 9, SafeLoad<uint32_t>(in + 9) >> 22 | SafeLoad<uint32_t>(in + 10) << 10, SafeLoad<uint32_t>(in + 10) >> 21 | SafeLoad<uint32_t>(in + 11) << 11, SafeLoad<uint32_t>(in + 11) >> 20 | SafeLoad<uint32_t>(in + 12) << 12, SafeLoad<uint32_t>(in + 12) >> 19 | SafeLoad<uint32_t>(in + 13) << 13, SafeLoad<uint32_t>(in + 13) >> 18 | SafeLoad<uint32_t>(in + 14) << 14, SafeLoad<uint32_t>(in + 14) >> 17 | SafeLoad<uint32_t>(in + 15) << 15 };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ // extract 31-bit bundles 16 to 31
+ words = simd_batch{ SafeLoad<uint32_t>(in + 15) >> 16 | SafeLoad<uint32_t>(in + 16) << 16, SafeLoad<uint32_t>(in + 16) >> 15 | SafeLoad<uint32_t>(in + 17) << 17, SafeLoad<uint32_t>(in + 17) >> 14 | SafeLoad<uint32_t>(in + 18) << 18, SafeLoad<uint32_t>(in + 18) >> 13 | SafeLoad<uint32_t>(in + 19) << 19, SafeLoad<uint32_t>(in + 19) >> 12 | SafeLoad<uint32_t>(in + 20) << 20, SafeLoad<uint32_t>(in + 20) >> 11 | SafeLoad<uint32_t>(in + 21) << 21, SafeLoad<uint32_t>(in + 21) >> 10 | SafeLoad<uint32_t>(in + 22) << 22, SafeLoad<uint32_t>(in + 22) >> 9 | SafeLoad<uint32_t>(in + 23) << 23, SafeLoad<uint32_t>(in + 23) >> 8 | SafeLoad<uint32_t>(in + 24) << 24, SafeLoad<uint32_t>(in + 24) >> 7 | SafeLoad<uint32_t>(in + 25) << 25, SafeLoad<uint32_t>(in + 25) >> 6 | SafeLoad<uint32_t>(in + 26) << 26, SafeLoad<uint32_t>(in + 26) >> 5 | SafeLoad<uint32_t>(in + 27) << 27, SafeLoad<uint32_t>(in + 27) >> 4 | SafeLoad<uint32_t>(in + 28) << 28, SafeLoad<uint32_t>(in + 28) >> 3 | SafeLoad<uint32_t>(in + 29) << 29, SafeLoad<uint32_t>(in + 29) >> 2 | SafeLoad<uint32_t>(in + 30) << 30, SafeLoad<uint32_t>(in + 30) };
+ shifts = simd_batch{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 };
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += 16;
+
+ in += 31;
+ return in;
+}
+
+inline static const uint32_t* unpack32_32(const uint32_t* in, uint32_t* out) {
+ memcpy(out, in, 32 * sizeof(*out));
+ in += 32;
+ out += 32;
+
+ return in;
+}
+
+}; // struct UnpackBits512
+
+} // namespace
+} // namespace internal
+} // namespace arrow
+
diff --git a/cpp/src/arrow/util/bpacking_simd_codegen.py b/cpp/src/arrow/util/bpacking_simd_codegen.py
new file mode 100644
index 0000000..d033394
--- /dev/null
+++ b/cpp/src/arrow/util/bpacking_simd_codegen.py
@@ -0,0 +1,209 @@
+#!/bin/python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Usage:
+# python bpacking_simd_codegen.py 128 > bpacking_simd128_generated.h
+# python bpacking_simd_codegen.py 256 > bpacking_simd256_generated.h
+# python bpacking_simd_codegen.py 512 > bpacking_simd512_generated.h
+
+from functools import partial
+import sys
+from textwrap import dedent, indent
+
+
+class UnpackGenerator:
+
+ def __init__(self, simd_width):
+ self.simd_width = simd_width
+ if simd_width % 32 != 0:
+ raise("SIMD bit width should be a multiple of 32")
+ self.simd_byte_width = simd_width // 8
+
+ def print_unpack_bit0_func(self):
+ print(
+ "inline static const uint32_t* unpack0_32(const uint32_t* in, uint32_t* out) {")
+ print(" memset(out, 0x0, 32 * sizeof(*out));")
+ print(" out += 32;")
+ print("")
+ print(" return in;")
+ print("}")
+
+
+ def print_unpack_bit32_func(self):
+ print(
+ "inline static const uint32_t* unpack32_32(const uint32_t* in, uint32_t* out) {")
+ print(" memcpy(out, in, 32 * sizeof(*out));")
+ print(" in += 32;")
+ print(" out += 32;")
+ print("")
+ print(" return in;")
+ print("}")
+
+ def print_unpack_bit_func(self, bit):
+ def p(code):
+ print(indent(code, prefix=' '))
+
+ shift = 0
+ shifts = []
+ in_index = 0
+ inls = []
+ mask = (1 << bit) - 1
+ bracket = "{"
+
+ print(f"inline static const uint32_t* unpack{bit}_32(const uint32_t* in, uint32_t* out) {{")
+ p(dedent(f"""\
+ uint32_t mask = 0x{mask:0x};
+
+ simd_batch masks(mask);
+ simd_batch words, shifts;
+ simd_batch results;
+ """))
+
+ def safe_load(index):
+ return f"SafeLoad<uint32_t>(in + {index})"
+
+ for i in range(32):
+ if shift + bit == 32:
+ shifts.append(shift)
+ inls.append(safe_load(in_index))
+ in_index += 1
+ shift = 0
+ elif shift + bit > 32: # cross the boundary
+ inls.append(
+ f"{safe_load(in_index)} >> {shift} | {safe_load(in_index + 1)} << {32 - shift}")
+ in_index += 1
+ shift = bit - (32 - shift)
+ shifts.append(0) # zero shift
+ else:
+ shifts.append(shift)
+ inls.append(safe_load(in_index))
+ shift += bit
+
+ bytes_per_batch = self.simd_byte_width
+ words_per_batch = bytes_per_batch // 4
+
+ one_word_template = dedent("""\
+ words = simd_batch{{ {words} }};
+ shifts = simd_batch{{ {shifts} }};
+ results = (words >> shifts) & masks;
+ results.store_unaligned(out);
+ out += {words_per_batch};
+ """)
+
+ for start in range(0, 32, words_per_batch):
+ stop = start + words_per_batch;
+ p(f"""// extract {bit}-bit bundles {start} to {stop - 1}""")
+ p(one_word_template.format(
+ words=", ".join(inls[start:stop]),
+ shifts=", ".join(map(str, shifts[start:stop])),
+ words_per_batch=words_per_batch))
+
+ p(dedent(f"""\
+ in += {bit};
+ return in;"""))
+ print("}")
+
+
+def print_copyright():
+ print(dedent("""\
+ // Licensed to the Apache Software Foundation (ASF) under one
+ // or more contributor license agreements. See the NOTICE file
+ // distributed with this work for additional information
+ // regarding copyright ownership. The ASF licenses this file
+ // to you under the Apache License, Version 2.0 (the
+ // "License"); you may not use this file except in compliance
+ // with the License. You may obtain a copy of the License at
+ //
+ // http://www.apache.org/licenses/LICENSE-2.0
+ //
+ // Unless required by applicable law or agreed to in writing,
+ // software distributed under the License is distributed on an
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ // KIND, either express or implied. See the License for the
+ // specific language governing permissions and limitations
+ // under the License.
+ """))
+
+
+def print_note():
+ print("// Automatically generated file; DO NOT EDIT.")
+ print()
+
+
+def main(simd_width):
+ print_copyright()
+ print_note()
+
+ struct_name = f"UnpackBits{simd_width}"
+
+ # NOTE: templating the UnpackBits struct on the dispatch level avoids
+ # potential name collisions if there are several UnpackBits generations
+ # with the same SIMD width on a given architecture.
+
+ print(dedent(f"""\
+ #pragma once
+
+ #include <cstdint>
+ #include <cstring>
+
+ #include <xsimd/xsimd.hpp>
+
+ #include "arrow/util/dispatch.h"
+ #include "arrow/util/ubsan.h"
+
+ namespace arrow {{
+ namespace internal {{
+ namespace {{
+
+ using ::arrow::util::SafeLoad;
+
+ template <DispatchLevel level>
+ struct {struct_name} {{
+
+ using simd_batch = xsimd::batch<uint32_t, {simd_width // 32}>;
+ """))
+
+ gen = UnpackGenerator(simd_width)
+ gen.print_unpack_bit0_func()
+ print()
+ for i in range(1, 32):
+ gen.print_unpack_bit_func(i)
+ print()
+ gen.print_unpack_bit32_func()
+ print()
+
+ print(dedent(f"""\
+ }}; // struct {struct_name}
+
+ }} // namespace
+ }} // namespace internal
+ }} // namespace arrow
+ """))
+
+
+if __name__ == '__main__':
+ usage = f"""Usage: {__file__} <SIMD bit-width>"""
+ if len(sys.argv) != 2:
+ raise ValueError(usage)
+ try:
+ simd_width = int(sys.argv[1])
+ except ValueError:
+ raise ValueError(usage)
+
+ main(simd_width)
diff --git a/cpp/src/arrow/util/bpacking_simd_internal.h b/cpp/src/arrow/util/bpacking_simd_internal.h
new file mode 100644
index 0000000..72d23f2
--- /dev/null
+++ b/cpp/src/arrow/util/bpacking_simd_internal.h
@@ -0,0 +1,138 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/util/dispatch.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+namespace internal {
+
+template <typename UnpackBits>
+static int unpack32_specialized(const uint32_t* in, uint32_t* out, int batch_size,
+ int num_bits) {
+ batch_size = batch_size / 32 * 32;
+ int num_loops = batch_size / 32;
+
+ switch (num_bits) {
+ case 0:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack0_32(in, out + i * 32);
+ break;
+ case 1:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack1_32(in, out + i * 32);
+ break;
+ case 2:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack2_32(in, out + i * 32);
+ break;
+ case 3:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack3_32(in, out + i * 32);
+ break;
+ case 4:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack4_32(in, out + i * 32);
+ break;
+ case 5:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack5_32(in, out + i * 32);
+ break;
+ case 6:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack6_32(in, out + i * 32);
+ break;
+ case 7:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack7_32(in, out + i * 32);
+ break;
+ case 8:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack8_32(in, out + i * 32);
+ break;
+ case 9:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack9_32(in, out + i * 32);
+ break;
+ case 10:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack10_32(in, out + i * 32);
+ break;
+ case 11:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack11_32(in, out + i * 32);
+ break;
+ case 12:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack12_32(in, out + i * 32);
+ break;
+ case 13:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack13_32(in, out + i * 32);
+ break;
+ case 14:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack14_32(in, out + i * 32);
+ break;
+ case 15:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack15_32(in, out + i * 32);
+ break;
+ case 16:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack16_32(in, out + i * 32);
+ break;
+ case 17:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack17_32(in, out + i * 32);
+ break;
+ case 18:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack18_32(in, out + i * 32);
+ break;
+ case 19:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack19_32(in, out + i * 32);
+ break;
+ case 20:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack20_32(in, out + i * 32);
+ break;
+ case 21:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack21_32(in, out + i * 32);
+ break;
+ case 22:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack22_32(in, out + i * 32);
+ break;
+ case 23:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack23_32(in, out + i * 32);
+ break;
+ case 24:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack24_32(in, out + i * 32);
+ break;
+ case 25:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack25_32(in, out + i * 32);
+ break;
+ case 26:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack26_32(in, out + i * 32);
+ break;
+ case 27:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack27_32(in, out + i * 32);
+ break;
+ case 28:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack28_32(in, out + i * 32);
+ break;
+ case 29:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack29_32(in, out + i * 32);
+ break;
+ case 30:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack30_32(in, out + i * 32);
+ break;
+ case 31:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack31_32(in, out + i * 32);
+ break;
+ case 32:
+ for (int i = 0; i < num_loops; ++i) in = UnpackBits::unpack32_32(in, out + i * 32);
+ break;
+ default:
+ DCHECK(false) << "Unsupported num_bits";
+ }
+
+ return batch_size;
+}
+
+} // namespace internal
+} // namespace arrow
diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt
index 01b658d..e5ab78c 100644
--- a/cpp/thirdparty/versions.txt
+++ b/cpp/thirdparty/versions.txt
@@ -52,8 +52,7 @@
ARROW_THRIFT_BUILD_VERSION=0.13.0
ARROW_THRIFT_BUILD_MD5_CHECKSUM=38a27d391a2b03214b444cb13d5664f1
ARROW_UTF8PROC_BUILD_VERSION=v2.6.1
-# For https://github.com/xtensor-stack/xsimd/pull/419
-ARROW_XSIMD_BUILD_VERSION=e916f3ab1bc513328b627df702226a1d1e2ae3a9
+ARROW_XSIMD_BUILD_VERSION=e9234cd6e6f4428fc260073b2c34ffe86fda1f34
ARROW_ZLIB_BUILD_VERSION=1.2.11
ARROW_ZSTD_BUILD_VERSION=v1.4.8
