// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// single_thread_gemm.h: programatically generated GEMM library header.

#ifndef GEMMLOWP_META_SINGLE_THREAD_GEMM_H_
#define GEMMLOWP_META_SINGLE_THREAD_GEMM_H_

#if defined(GEMMLOWP_NEON_32) || defined(GEMMLOWP_NEON_64)

#include <cassert>

#if defined(GEMMLOWP_NEON_32)
#include "single_thread_gemm_arm32.h"
#elif defined(GEMMLOWP_NEON_64)
#include "single_thread_gemm_arm64.h"
#endif

namespace gemmlowp {
namespace meta {
namespace internal {

void gemm_q8_0_0_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_0_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_0_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_0_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_0_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_0_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_0_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_0_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_1_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_1_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_1_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_1_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_1_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_1_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_1_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_1_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_2_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_2_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_2_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_2_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_2_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_2_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_2_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_2_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_1_0_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_0_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_0_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_0_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_0_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_0_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_0_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_0_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_1_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_1_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_1_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_1_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_1_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_1_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_1_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_1_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_2_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_2_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_2_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_2_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_2_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_2_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_2_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_2_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_1_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_0_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_0_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_0_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_0_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_0_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_0_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_0_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_0_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_1_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_1_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_1_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_1_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_1_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_1_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_1_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_1_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_2_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_2_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_2_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_2_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_2_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_2_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_2_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_2_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                           const std::uint8_t* rhs, std::int32_t m,
                           std::int32_t n, std::int32_t k,
                           std::int32_t lhs_offset, std::int32_t rhs_offset,
                           std::int32_t result_offset,
                           std::int32_t multiplicative_offset,
                           std::int32_t shift, std::uint8_t* result,
                           std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                          zipped_lhs_3_offsets, result_chunk, result_stride,
                          multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8_aligned(temp_result, n, mul_result_chunk_stride_bytes,
                        zipped_lhs_2_offsets, result_chunk, result_stride,
                        multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_0_0_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_0_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_0_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_0_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_0_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_0_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_0_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_0_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_1_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_1_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_1_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_1_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_1_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_1_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_1_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_1_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_2_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_2_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_2_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_2_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_2_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_2_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_2_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_0_2_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_q8_1_0_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_0_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_0_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_0_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_0_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_0_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_0_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_0_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_1_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_1_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_1_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_1_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_1_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_1_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_1_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_1_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_2_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_2_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_2_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_2_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_2_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_2_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_2_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_1_2_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_1x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_1_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_0_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_0_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_0_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_0_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_0_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_0_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_0_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_0_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_1_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_1_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_1_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_1_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_1_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_1_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_1_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_1_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_2_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_2_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_2_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_2_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_2_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_2_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_2_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_q8_2_2_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                   const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                   std::int32_t k, std::int32_t lhs_offset,
                   std::int32_t rhs_offset, std::int32_t result_offset,
                   std::int32_t multiplicative_offset, std::int32_t shift,
                   std::uint8_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result = reinterpret_cast<std::int32_t*>(
      zipped_rhs + ((zipped_rhs_size + 15) / 16) * 16);
  std::uint8_t* result_chunk = result;
  std::int32_t* mul_result_chunk = temp_result;
  const std::int32_t mul_result_chunk_stride_bytes = ((n * 4 + 31) / 32) * 32;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = temp_result;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                               mul_result_chunk, mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    multi_qnt_3x8(temp_result, n, mul_result_chunk_stride_bytes,
                  zipped_lhs_3_offsets, result_chunk, result_stride,
                  multiplicative_offset, rounding_offset, -shift);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = temp_result;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                             mul_result_chunk, mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                           mul_result_chunk, mul_result_chunk_stride_bytes);
  multi_qnt_2x8(temp_result, n, mul_result_chunk_stride_bytes,
                zipped_lhs_2_offsets, result_chunk, result_stride,
                multiplicative_offset, rounding_offset, -shift);
}

void gemm_i32_0_0_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_0_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_0_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_0_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_0_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_0_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_0_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_0_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_1_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_1_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_1_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_1_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_1_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_1_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_1_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_1_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_2_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_2_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_2_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_2_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_2_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_2_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_2_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_2_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_1_0_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_1_0_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_1_0_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_1_0_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_1_0_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_1_0_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_1_0_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_1_0_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_1_1_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_1_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_1_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_1_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_1_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_1_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_1_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_1_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_2_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_2_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_2_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_2_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_2_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_2_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_2_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_2_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_0_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_2_0_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_2_0_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_2_0_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_2_0_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_2_0_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_2_0_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_2_0_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_2_1_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_1_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_1_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_1_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_1_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_1_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_1_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_1_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_2_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_2_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_2_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_2_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_2_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_2_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_2_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_2_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                            const std::uint8_t* rhs, std::int32_t m,
                            std::int32_t n, std::int32_t k,
                            std::int32_t lhs_offset, std::int32_t rhs_offset,
                            std::int32_t* result, std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_0_0_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_0_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_0_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_0_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_0_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_0_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_0_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_0_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_1_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_1_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_1_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_1_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_1_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_1_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_1_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_1_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_2_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_2_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_2_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_2_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_2_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_2_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_2_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_0_2_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_i32_1_0_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_1_0_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_1_0_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_1_0_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_1_0_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_1_0_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_1_0_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_1_0_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_1_1_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_1_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_1_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_1_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_1_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_1_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_1_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_1_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_2_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_2_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_2_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_2_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_2_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_2_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_2_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_1_2_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_0_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_2_0_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_2_0_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_2_0_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_2_0_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_2_0_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_2_0_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_2_0_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_i32_2_1_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_1_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_1_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_1_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_1_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_1_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_1_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_1_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_2_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_2_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_2_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_2_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_2_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_2_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_2_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_i32_2_2_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, std::int32_t* result,
                    std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* result_chunk = result;
  std::int32_t* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                      mul_result_chunk,
                                      mul_result_chunk_stride_bytes);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                    mul_result_chunk,
                                    mul_result_chunk_stride_bytes);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes);
}

void gemm_f_0_0_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_0_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_0_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_0_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_0_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_0_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_0_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_0_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_1_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_1_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_1_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_1_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_1_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_1_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_1_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_1_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_2_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_2_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_2_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_2_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_2_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_2_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_2_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_2_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_1_0_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_1_0_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_1_0_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_1_0_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_1_0_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_1_0_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_1_0_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_1_0_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_1_1_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_1_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_1_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_1_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_1_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_1_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_1_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_1_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_2_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_2_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_2_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_2_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_2_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_2_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_2_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_2_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_0_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_2_0_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_2_0_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_2_0_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_2_0_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_2_0_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_2_0_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_2_0_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_2_1_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_1_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_1_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_1_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_1_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_1_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_1_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_1_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_2_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_2_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_2_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_2_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_2_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_2_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_2_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_2_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t m,
                          std::int32_t n, std::int32_t k,
                          std::int32_t lhs_offset, std::int32_t rhs_offset,
                          float result_scale, float* result,
                          std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7_aligned(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7_aligned(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_0_0_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_0_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_0_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_0_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_0_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_0_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_0_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_0_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_1_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_1_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_1_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_1_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_1_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_1_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_1_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_1_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_2_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_2_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_2_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_2_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_2_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_2_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_2_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_0_2_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }
}

void gemm_f_1_0_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_1_0_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_1_0_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_1_0_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_1_0_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_1_0_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_1_0_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_1_0_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_1_1_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_1_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_1_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_1_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_1_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_1_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_1_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_1_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_2_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_2_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_2_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_2_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_2_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_2_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_2_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_1_2_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_1_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 1);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_1x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_1x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_0_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_2_0_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_2_0_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_2_0_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_2_0_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_2_0_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_2_0_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_2_0_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
}

void gemm_f_2_1_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_1_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_1_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_1_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_1_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_1_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_1_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_1_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_1x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_1x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_2_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_2_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_1(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_1(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_2_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_2(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_2(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_2_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_3(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_3(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_2_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_4(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_4(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_2_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_5(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_5(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_2_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_6(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_6(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemm_f_2_2_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                  std::int32_t k, std::int32_t lhs_offset,
                  std::int32_t rhs_offset, float result_scale, float* result,
                  std::int32_t result_stride) {
  const std::int32_t row_chunks = m / 3;
  const std::int32_t col_chunks = n / 3;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 3;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 3;
  const std::int32_t zipped_rhs_size = (padded_k + 16) * n;
  const std::uint8_t* lhs_chunk = lhs;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_3_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 3);
  std::int32_t* zipped_lhs_2_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k * 2);
  std::uint8_t* zipped_rhs = scratch + ((zipped_chunk_size + 15) / 16) * 16;
  std::uint8_t* zipped_rhs_chunk = zipped_rhs;
  const std::int32_t result_chunk_stride = result_stride * 3;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* result_chunk = result;
  float* mul_result_chunk = result;
  const std::int32_t mul_result_chunk_stride_bytes = result_stride * 4;

  for (int i = 0; i < col_chunks; ++i) {
    zip_3x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);
    rhs_chunk += chunk_size;
    zipped_rhs_chunk += zipped_chunk_size;
  }
  zip_2x8_7(rhs_chunk, k, k, zipped_rhs_chunk, lhs_offset, 0);

  for (int i = 0; i < row_chunks; ++i) {
    zip_3x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
    zipped_rhs_chunk = zipped_rhs;
    mul_result_chunk = result_chunk;
    for (int j = 0; j < col_chunks; ++j) {
      mul_3x8_3x8_float_lhsadd_rhsadd(
          zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
          mul_result_chunk_stride_bytes, result_scale);
      zipped_rhs_chunk += zipped_chunk_size;
      mul_result_chunk += 3;
    }
    mul_3x8_2x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    lhs_chunk += chunk_size;
    result_chunk += result_chunk_stride;
  }

  zip_2x8_7(lhs_chunk, k, k, zipped_lhs, rhs_offset, const_offset);
  zipped_rhs_chunk = zipped_rhs;
  mul_result_chunk = result_chunk;
  for (int j = 0; j < col_chunks; ++j) {
    mul_2x8_3x8_float_lhsadd_rhsadd(
        zipped_lhs, zipped_rhs_chunk, padded_k, mul_result_chunk,
        mul_result_chunk_stride_bytes, result_scale);
    zipped_rhs_chunk += zipped_chunk_size;
    mul_result_chunk += 3;
  }
  mul_2x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_chunk, padded_k,
                                  mul_result_chunk,
                                  mul_result_chunk_stride_bytes, result_scale);
}

void gemv_q8_0_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  qnt_1x8_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                  multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_0_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  qnt_1x8_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                  multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_0_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  qnt_1x8_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                  multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_0_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  qnt_1x8_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                  multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_0_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  qnt_1x8_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                  multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_0_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  qnt_1x8_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                  multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_0_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  qnt_1x8_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                  multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_0_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  qnt_1x8_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                  multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_1_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_1_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_1_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_1_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_1_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_1_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_1_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_1_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_1_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_1_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_1_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_1_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_1_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_1_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_1_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_1_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_2_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_2_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_2_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_2_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_2_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_2_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_2_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_2_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_2_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_2_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_2_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_2_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_2_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_2_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_2_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_2_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_3_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_3_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_3_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_3_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_3_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_3_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_3_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_3_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_3_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_3_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_3_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_3_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_3_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_3_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_3_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_3_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_4_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_4_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_4_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_4_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_4_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_4_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_4_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_4_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_4_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_4_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_4_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_4_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_4_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_4_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_4_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_4_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_5_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_5_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_5_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_5_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_5_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_5_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_5_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_5_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_5_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_5_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_5_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_5_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_5_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_5_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_5_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_5_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_6_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_6_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_6_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_6_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_6_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_6_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_6_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_6_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_6_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_6_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_6_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_6_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_6_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_6_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_6_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_6_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_7_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_7_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_7_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_7_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_7_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_7_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_7_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_7_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_7_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_7_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_7_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_7_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_7_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_7_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_7_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                         const std::uint8_t* rhs, std::int32_t n,
                         std::int32_t k, std::int32_t lhs_offset,
                         std::int32_t rhs_offset, std::int32_t result_offset,
                         std::int32_t multiplicative_offset, std::int32_t shift,
                         std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_7_aligned(temp_result, n, 0, zipped_lhs_offsets, result, 0,
                    multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_0_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  qnt_1x8(temp_result, n, 0, zipped_lhs_offsets, result, 0,
          multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_0_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  qnt_1x8(temp_result, n, 0, zipped_lhs_offsets, result, 0,
          multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_0_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  qnt_1x8(temp_result, n, 0, zipped_lhs_offsets, result, 0,
          multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_0_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  qnt_1x8(temp_result, n, 0, zipped_lhs_offsets, result, 0,
          multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_0_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  qnt_1x8(temp_result, n, 0, zipped_lhs_offsets, result, 0,
          multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_0_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  qnt_1x8(temp_result, n, 0, zipped_lhs_offsets, result, 0,
          multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_0_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  qnt_1x8(temp_result, n, 0, zipped_lhs_offsets, result, 0,
          multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_0_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  qnt_1x8(temp_result, n, 0, zipped_lhs_offsets, result, 0,
          multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_1_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_1(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_1_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_1(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_1_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_1(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_1_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_1(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_1_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_1(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_1_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_1(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_1_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_1(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_1_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_1(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_2_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_2(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_2_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_2(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_2_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_2(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_2_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_2(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_2_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_2(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_2_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_2(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_2_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_2(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_2_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_2(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_3_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_3(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_3_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_3(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_3_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_3(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_3_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_3(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_3_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_3(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_3_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_3(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_3_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_3(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_3_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_3(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_4_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_4(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_4_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_4(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_4_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_4(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_4_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_4(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_4_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_4(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_4_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_4(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_4_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_4(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_4_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, padded_k, mul_result_chunk,
                           0);
  qnt_1x8_4(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_5_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_5(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_5_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_5(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_5_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_5(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_5_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_5(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_5_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_5(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_5_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_5(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_5_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_5(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_5_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_5(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_6_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_6(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_6_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_6(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_6_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_6(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_6_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_6(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_6_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_6(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_6_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_6(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_6_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_6(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_6_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_6(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_7_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_7(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_7_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_7(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_7_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_7(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_7_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_7(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_7_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_7(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_7_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_7(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_7_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_7(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_q8_7_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                 const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                 std::int32_t lhs_offset, std::int32_t rhs_offset,
                 std::int32_t result_offset, std::int32_t multiplicative_offset,
                 std::int32_t shift, std::uint8_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k + result_offset;
  const std::int32_t rounding_offset = (1 << (shift - 1));
  std::int32_t* temp_result =
      reinterpret_cast<std::int32_t*>(zipped_rhs_2 + zipped_chunk_size);
  std::int32_t* mul_result_chunk = temp_result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                             mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2, padded_k,
                           mul_result_chunk);
  qnt_1x8_7(temp_result, n, 0, zipped_lhs_offsets, result, 0,
            multiplicative_offset, rounding_offset, -shift);
}

void gemv_i32_0_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
}

void gemv_i32_0_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
}

void gemv_i32_0_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
}

void gemv_i32_0_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
}

void gemv_i32_0_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
}

void gemv_i32_0_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
}

void gemv_i32_0_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
}

void gemv_i32_0_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
}

void gemv_i32_1_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_1_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_1_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_1_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_1_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_1_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_1_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_1_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_2_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_2_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_2_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_2_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_2_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_2_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_2_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_2_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_3_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_3_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_3_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_3_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_3_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_3_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_3_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_3_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_4_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_4_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_4_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_4_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_4_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_4_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_4_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_4_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_5_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_5_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_5_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_5_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_5_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_5_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_5_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_5_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_6_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_6_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_6_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_6_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_6_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_6_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_6_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_6_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_7_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_7_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_7_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_7_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_7_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_7_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_7_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_7_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                          const std::uint8_t* rhs, std::int32_t n,
                          std::int32_t k, std::int32_t lhs_offset,
                          std::int32_t rhs_offset, std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_0_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
}

void gemv_i32_0_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
}

void gemv_i32_0_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
}

void gemv_i32_0_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
}

void gemv_i32_0_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
}

void gemv_i32_0_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
}

void gemv_i32_0_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
}

void gemv_i32_0_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
}

void gemv_i32_1_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_1_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_1_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_1_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_1_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_1_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_1_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_1_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_1x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_2_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_2_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_2_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_2_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_2_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_2_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_2_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_2_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_2x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_3_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_3_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_3_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_3_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_3_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_3_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_3_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_3_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_3x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_4_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_4_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_4_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_4_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_4_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_4_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_4_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_4_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0);
}

void gemv_i32_5_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_5_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_5_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_5_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_5_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_5_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_5_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_5_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_6_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_6_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_6_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_6_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_6_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_6_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_6_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_6_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_7_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_7_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_7_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_7_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_7_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_7_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_7_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_i32_7_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                  const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                  std::int32_t lhs_offset, std::int32_t rhs_offset,
                  std::int32_t* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  std::int32_t* mul_result_chunk = result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk);
    mul_result_chunk += 8;
  }
  zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_int32_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk);
}

void gemv_f_0_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
}

void gemv_f_0_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
}

void gemv_f_0_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
}

void gemv_f_0_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
}

void gemv_f_0_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
}

void gemv_f_0_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
}

void gemv_f_0_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
}

void gemv_f_0_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
}

void gemv_f_1_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_1x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_1_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_1x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_1_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_1x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_1_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_1x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_1_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_1x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_1_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_1x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_1_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_1x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_1_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_1x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_2_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_2x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_2_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_2x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_2_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_2x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_2_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_2x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_2_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_2x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_2_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_2x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_2_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_2x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_2_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_2x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_3_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_3_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_3_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_3_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_3_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_3_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_3_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_3_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_4_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_4_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_4_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_4_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_4_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_4_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_4_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_4_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_5_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_5_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_5_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_5_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_5_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_5_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_5_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_5_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_6_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_6_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_6_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_6_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_6_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_6_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_6_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_6_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_7_0_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_7_1_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_1_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_1_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_1_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_7_2_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_2_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_2_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_2_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_7_3_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_3_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_3_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_3_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_7_4_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_4_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_4_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_4_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_7_5_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_5_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_5_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_5_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_7_6_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_6_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_6_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_6_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_7_7_aligned(std::uint8_t* scratch, const std::uint8_t* lhs,
                        const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                        std::int32_t lhs_offset, std::int32_t rhs_offset,
                        float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_7_aligned(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_7_aligned(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_7_aligned(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_0_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
}

void gemv_f_0_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
}

void gemv_f_0_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
}

void gemv_f_0_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
}

void gemv_f_0_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
}

void gemv_f_0_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
}

void gemv_f_0_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
}

void gemv_f_0_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
}

void gemv_f_1_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_1x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_1_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_1x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_1_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_1x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_1_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_1x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_1_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_1x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_1_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_1x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_1_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_1x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_1_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_1x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_1x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_2_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_2x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_2_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_2x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_2_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_2x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_2_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_2x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_2_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_2x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_2_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_2x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_2_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_2x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_2_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_2x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_2x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_3_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_3x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_3_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_3x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_3_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_3x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_3_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_3x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_3_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_3x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_3_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_3x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_3_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_3x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_3_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_3x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_3x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_4_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_4_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_4_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_4_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_4_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_4_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_4_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_4_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  mul_1x8_4x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, padded_k,
                                  mul_result_chunk, 0, result_scale);
}

void gemv_f_5_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_5_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_5_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_5_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_5_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_5_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_5_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_5_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_1x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_5x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_6_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_6_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_6_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_6_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_6_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_6_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_6_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_6_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_2x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_6x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_7_0(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_7_1(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_1(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_1(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_1(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_7_2(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_2(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_2(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_2(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_7_3(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_3(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_3(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_3(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_7_4(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_4(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_4(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_4(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_7_5(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_5(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_5(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_5(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_7_6(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_6(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_6(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_6(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

void gemv_f_7_7(std::uint8_t* scratch, const std::uint8_t* lhs,
                const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
                std::int32_t lhs_offset, std::int32_t rhs_offset,
                float result_scale, float* result) {
  const std::int32_t col_chunks = n / 8;
  const std::int32_t padded_k = ((k + 7) / 8) * 8;
  const std::int32_t chunk_size = k * 4;
  const std::int32_t zipped_chunk_size = (padded_k + 16) * 4;
  const std::uint8_t* rhs_chunk = rhs;
  std::uint8_t* zipped_lhs = scratch;
  std::int32_t* zipped_lhs_offsets =
      reinterpret_cast<std::int32_t*>(zipped_lhs + padded_k);
  std::uint8_t* zipped_rhs_1 = scratch + ((padded_k + 31) / 16) * 16;
  std::uint8_t* zipped_rhs_2 = zipped_rhs_1 + zipped_chunk_size;

  const std::int32_t const_offset = lhs_offset * rhs_offset * k;
  float* mul_result_chunk = result;

  zip_1x8_7(lhs, k, k, zipped_lhs, rhs_offset, 0);
  for (int i = 0; i < col_chunks; ++i) {
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    zip_4x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
    rhs_chunk += chunk_size;
    mul_1x8_8x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                    padded_k, mul_result_chunk, result_scale);
    mul_result_chunk += 8;
  }
  zip_4x8_7(rhs_chunk, k, k, zipped_rhs_1, lhs_offset, const_offset);
  rhs_chunk += chunk_size;
  zip_3x8_7(rhs_chunk, k, k, zipped_rhs_2, lhs_offset, const_offset);
  mul_1x8_7x8_float_lhsadd_rhsadd(zipped_lhs, zipped_rhs_1, zipped_rhs_2,
                                  padded_k, mul_result_chunk, result_scale);
}

}  // namespace internal

void gemm_q8_strided(std::uint8_t* scratch, const std::uint8_t* lhs,
                     const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                     std::int32_t k, std::int32_t lhs_offset,
                     std::int32_t rhs_offset, std::int32_t result_offset,
                     std::int32_t multiplicative_offset, std::int32_t shift,
                     std::uint8_t* result, std::int32_t result_stride) {
  const bool lhs_aligned = ((reinterpret_cast<std::uintptr_t>(lhs) % 8) == 0);
  const bool rhs_aligned = ((reinterpret_cast<std::uintptr_t>(rhs) % 8) == 0);
  const bool k_aligned = ((k % 8) == 0);
  const bool result_aligned =
      ((reinterpret_cast<std::uintptr_t>(result) % 8) == 0);
  const bool result_stride_aligned = ((result_stride % 8) == 0);
  const bool aligned = lhs_aligned && rhs_aligned && result_aligned &&
                       k_aligned && result_stride_aligned;
  if (aligned) {
    switch (m % 3) {
      case 0:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_0_0_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 1:
                internal::gemm_q8_0_0_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 2:
                internal::gemm_q8_0_0_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 3:
                internal::gemm_q8_0_0_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 4:
                internal::gemm_q8_0_0_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 5:
                internal::gemm_q8_0_0_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 6:
                internal::gemm_q8_0_0_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 7:
                internal::gemm_q8_0_0_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_0_1_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 1:
                internal::gemm_q8_0_1_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 2:
                internal::gemm_q8_0_1_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 3:
                internal::gemm_q8_0_1_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 4:
                internal::gemm_q8_0_1_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 5:
                internal::gemm_q8_0_1_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 6:
                internal::gemm_q8_0_1_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 7:
                internal::gemm_q8_0_1_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_0_2_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 1:
                internal::gemm_q8_0_2_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 2:
                internal::gemm_q8_0_2_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 3:
                internal::gemm_q8_0_2_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 4:
                internal::gemm_q8_0_2_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 5:
                internal::gemm_q8_0_2_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 6:
                internal::gemm_q8_0_2_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 7:
                internal::gemm_q8_0_2_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
            }
            break;
        }
        break;
      case 1:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_1_0_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 1:
                internal::gemm_q8_1_0_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 2:
                internal::gemm_q8_1_0_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 3:
                internal::gemm_q8_1_0_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 4:
                internal::gemm_q8_1_0_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 5:
                internal::gemm_q8_1_0_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 6:
                internal::gemm_q8_1_0_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 7:
                internal::gemm_q8_1_0_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_1_1_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 1:
                internal::gemm_q8_1_1_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 2:
                internal::gemm_q8_1_1_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 3:
                internal::gemm_q8_1_1_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 4:
                internal::gemm_q8_1_1_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 5:
                internal::gemm_q8_1_1_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 6:
                internal::gemm_q8_1_1_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 7:
                internal::gemm_q8_1_1_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_1_2_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 1:
                internal::gemm_q8_1_2_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 2:
                internal::gemm_q8_1_2_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 3:
                internal::gemm_q8_1_2_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 4:
                internal::gemm_q8_1_2_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 5:
                internal::gemm_q8_1_2_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 6:
                internal::gemm_q8_1_2_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 7:
                internal::gemm_q8_1_2_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
            }
            break;
        }
        break;
      case 2:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_2_0_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 1:
                internal::gemm_q8_2_0_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 2:
                internal::gemm_q8_2_0_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 3:
                internal::gemm_q8_2_0_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 4:
                internal::gemm_q8_2_0_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 5:
                internal::gemm_q8_2_0_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 6:
                internal::gemm_q8_2_0_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 7:
                internal::gemm_q8_2_0_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_2_1_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 1:
                internal::gemm_q8_2_1_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 2:
                internal::gemm_q8_2_1_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 3:
                internal::gemm_q8_2_1_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 4:
                internal::gemm_q8_2_1_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 5:
                internal::gemm_q8_2_1_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 6:
                internal::gemm_q8_2_1_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 7:
                internal::gemm_q8_2_1_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_2_2_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 1:
                internal::gemm_q8_2_2_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 2:
                internal::gemm_q8_2_2_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 3:
                internal::gemm_q8_2_2_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 4:
                internal::gemm_q8_2_2_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 5:
                internal::gemm_q8_2_2_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 6:
                internal::gemm_q8_2_2_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
              case 7:
                internal::gemm_q8_2_2_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_offset, multiplicative_offset, shift, result,
                    result_stride);
                break;
            }
            break;
        }
        break;
    }
  } else {
    switch (m % 3) {
      case 0:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_0_0_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 1:
                internal::gemm_q8_0_0_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 2:
                internal::gemm_q8_0_0_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 3:
                internal::gemm_q8_0_0_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 4:
                internal::gemm_q8_0_0_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 5:
                internal::gemm_q8_0_0_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 6:
                internal::gemm_q8_0_0_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 7:
                internal::gemm_q8_0_0_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_0_1_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 1:
                internal::gemm_q8_0_1_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 2:
                internal::gemm_q8_0_1_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 3:
                internal::gemm_q8_0_1_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 4:
                internal::gemm_q8_0_1_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 5:
                internal::gemm_q8_0_1_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 6:
                internal::gemm_q8_0_1_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 7:
                internal::gemm_q8_0_1_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_0_2_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 1:
                internal::gemm_q8_0_2_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 2:
                internal::gemm_q8_0_2_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 3:
                internal::gemm_q8_0_2_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 4:
                internal::gemm_q8_0_2_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 5:
                internal::gemm_q8_0_2_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 6:
                internal::gemm_q8_0_2_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 7:
                internal::gemm_q8_0_2_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
            }
            break;
        }
        break;
      case 1:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_1_0_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 1:
                internal::gemm_q8_1_0_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 2:
                internal::gemm_q8_1_0_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 3:
                internal::gemm_q8_1_0_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 4:
                internal::gemm_q8_1_0_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 5:
                internal::gemm_q8_1_0_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 6:
                internal::gemm_q8_1_0_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 7:
                internal::gemm_q8_1_0_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_1_1_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 1:
                internal::gemm_q8_1_1_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 2:
                internal::gemm_q8_1_1_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 3:
                internal::gemm_q8_1_1_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 4:
                internal::gemm_q8_1_1_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 5:
                internal::gemm_q8_1_1_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 6:
                internal::gemm_q8_1_1_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 7:
                internal::gemm_q8_1_1_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_1_2_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 1:
                internal::gemm_q8_1_2_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 2:
                internal::gemm_q8_1_2_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 3:
                internal::gemm_q8_1_2_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 4:
                internal::gemm_q8_1_2_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 5:
                internal::gemm_q8_1_2_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 6:
                internal::gemm_q8_1_2_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 7:
                internal::gemm_q8_1_2_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
            }
            break;
        }
        break;
      case 2:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_2_0_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 1:
                internal::gemm_q8_2_0_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 2:
                internal::gemm_q8_2_0_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 3:
                internal::gemm_q8_2_0_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 4:
                internal::gemm_q8_2_0_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 5:
                internal::gemm_q8_2_0_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 6:
                internal::gemm_q8_2_0_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 7:
                internal::gemm_q8_2_0_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_2_1_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 1:
                internal::gemm_q8_2_1_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 2:
                internal::gemm_q8_2_1_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 3:
                internal::gemm_q8_2_1_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 4:
                internal::gemm_q8_2_1_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 5:
                internal::gemm_q8_2_1_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 6:
                internal::gemm_q8_2_1_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 7:
                internal::gemm_q8_2_1_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_q8_2_2_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 1:
                internal::gemm_q8_2_2_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 2:
                internal::gemm_q8_2_2_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 3:
                internal::gemm_q8_2_2_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 4:
                internal::gemm_q8_2_2_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 5:
                internal::gemm_q8_2_2_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 6:
                internal::gemm_q8_2_2_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
              case 7:
                internal::gemm_q8_2_2_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                        rhs_offset, result_offset,
                                        multiplicative_offset, shift, result,
                                        result_stride);
                break;
            }
            break;
        }
        break;
    }
  }
}

void gemm_q8(std::uint8_t* scratch, const std::uint8_t* lhs,
             const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
             std::int32_t k, std::int32_t lhs_offset, std::int32_t rhs_offset,
             std::int32_t result_offset, std::int32_t multiplicative_offset,
             std::int32_t shift, std::uint8_t* result) {
  gemm_q8_strided(scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                  result_offset, multiplicative_offset, shift, result, n);
}

void gemm_i32_strided(std::uint8_t* scratch, const std::uint8_t* lhs,
                      const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                      std::int32_t k, std::int32_t lhs_offset,
                      std::int32_t rhs_offset, std::int32_t* result,
                      std::int32_t result_stride) {
  const bool lhs_aligned = ((reinterpret_cast<std::uintptr_t>(lhs) % 8) == 0);
  const bool rhs_aligned = ((reinterpret_cast<std::uintptr_t>(rhs) % 8) == 0);
  const bool k_aligned = ((k % 8) == 0);
  const bool aligned = lhs_aligned && rhs_aligned && k_aligned;
  if (aligned) {
    switch (m % 3) {
      case 0:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_0_0_0_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 1:
                internal::gemm_i32_0_0_1_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 2:
                internal::gemm_i32_0_0_2_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 3:
                internal::gemm_i32_0_0_3_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 4:
                internal::gemm_i32_0_0_4_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 5:
                internal::gemm_i32_0_0_5_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 6:
                internal::gemm_i32_0_0_6_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 7:
                internal::gemm_i32_0_0_7_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_0_1_0_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 1:
                internal::gemm_i32_0_1_1_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 2:
                internal::gemm_i32_0_1_2_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 3:
                internal::gemm_i32_0_1_3_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 4:
                internal::gemm_i32_0_1_4_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 5:
                internal::gemm_i32_0_1_5_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 6:
                internal::gemm_i32_0_1_6_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 7:
                internal::gemm_i32_0_1_7_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_0_2_0_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 1:
                internal::gemm_i32_0_2_1_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 2:
                internal::gemm_i32_0_2_2_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 3:
                internal::gemm_i32_0_2_3_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 4:
                internal::gemm_i32_0_2_4_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 5:
                internal::gemm_i32_0_2_5_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 6:
                internal::gemm_i32_0_2_6_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 7:
                internal::gemm_i32_0_2_7_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
            }
            break;
        }
        break;
      case 1:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_1_0_0_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 1:
                internal::gemm_i32_1_0_1_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 2:
                internal::gemm_i32_1_0_2_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 3:
                internal::gemm_i32_1_0_3_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 4:
                internal::gemm_i32_1_0_4_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 5:
                internal::gemm_i32_1_0_5_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 6:
                internal::gemm_i32_1_0_6_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 7:
                internal::gemm_i32_1_0_7_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_1_1_0_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 1:
                internal::gemm_i32_1_1_1_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 2:
                internal::gemm_i32_1_1_2_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 3:
                internal::gemm_i32_1_1_3_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 4:
                internal::gemm_i32_1_1_4_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 5:
                internal::gemm_i32_1_1_5_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 6:
                internal::gemm_i32_1_1_6_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 7:
                internal::gemm_i32_1_1_7_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_1_2_0_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 1:
                internal::gemm_i32_1_2_1_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 2:
                internal::gemm_i32_1_2_2_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 3:
                internal::gemm_i32_1_2_3_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 4:
                internal::gemm_i32_1_2_4_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 5:
                internal::gemm_i32_1_2_5_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 6:
                internal::gemm_i32_1_2_6_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 7:
                internal::gemm_i32_1_2_7_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
            }
            break;
        }
        break;
      case 2:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_2_0_0_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 1:
                internal::gemm_i32_2_0_1_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 2:
                internal::gemm_i32_2_0_2_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 3:
                internal::gemm_i32_2_0_3_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 4:
                internal::gemm_i32_2_0_4_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 5:
                internal::gemm_i32_2_0_5_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 6:
                internal::gemm_i32_2_0_6_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 7:
                internal::gemm_i32_2_0_7_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_2_1_0_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 1:
                internal::gemm_i32_2_1_1_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 2:
                internal::gemm_i32_2_1_2_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 3:
                internal::gemm_i32_2_1_3_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 4:
                internal::gemm_i32_2_1_4_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 5:
                internal::gemm_i32_2_1_5_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 6:
                internal::gemm_i32_2_1_6_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 7:
                internal::gemm_i32_2_1_7_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_2_2_0_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 1:
                internal::gemm_i32_2_2_1_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 2:
                internal::gemm_i32_2_2_2_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 3:
                internal::gemm_i32_2_2_3_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 4:
                internal::gemm_i32_2_2_4_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 5:
                internal::gemm_i32_2_2_5_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 6:
                internal::gemm_i32_2_2_6_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
              case 7:
                internal::gemm_i32_2_2_7_aligned(scratch, lhs, rhs, m, n, k,
                                                 lhs_offset, rhs_offset, result,
                                                 result_stride);
                break;
            }
            break;
        }
        break;
    }
  } else {
    switch (m % 3) {
      case 0:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_0_0_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 1:
                internal::gemm_i32_0_0_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 2:
                internal::gemm_i32_0_0_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 3:
                internal::gemm_i32_0_0_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 4:
                internal::gemm_i32_0_0_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 5:
                internal::gemm_i32_0_0_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 6:
                internal::gemm_i32_0_0_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 7:
                internal::gemm_i32_0_0_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_0_1_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 1:
                internal::gemm_i32_0_1_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 2:
                internal::gemm_i32_0_1_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 3:
                internal::gemm_i32_0_1_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 4:
                internal::gemm_i32_0_1_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 5:
                internal::gemm_i32_0_1_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 6:
                internal::gemm_i32_0_1_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 7:
                internal::gemm_i32_0_1_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_0_2_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 1:
                internal::gemm_i32_0_2_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 2:
                internal::gemm_i32_0_2_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 3:
                internal::gemm_i32_0_2_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 4:
                internal::gemm_i32_0_2_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 5:
                internal::gemm_i32_0_2_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 6:
                internal::gemm_i32_0_2_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 7:
                internal::gemm_i32_0_2_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
            }
            break;
        }
        break;
      case 1:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_1_0_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 1:
                internal::gemm_i32_1_0_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 2:
                internal::gemm_i32_1_0_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 3:
                internal::gemm_i32_1_0_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 4:
                internal::gemm_i32_1_0_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 5:
                internal::gemm_i32_1_0_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 6:
                internal::gemm_i32_1_0_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 7:
                internal::gemm_i32_1_0_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_1_1_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 1:
                internal::gemm_i32_1_1_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 2:
                internal::gemm_i32_1_1_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 3:
                internal::gemm_i32_1_1_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 4:
                internal::gemm_i32_1_1_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 5:
                internal::gemm_i32_1_1_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 6:
                internal::gemm_i32_1_1_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 7:
                internal::gemm_i32_1_1_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_1_2_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 1:
                internal::gemm_i32_1_2_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 2:
                internal::gemm_i32_1_2_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 3:
                internal::gemm_i32_1_2_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 4:
                internal::gemm_i32_1_2_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 5:
                internal::gemm_i32_1_2_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 6:
                internal::gemm_i32_1_2_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 7:
                internal::gemm_i32_1_2_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
            }
            break;
        }
        break;
      case 2:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_2_0_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 1:
                internal::gemm_i32_2_0_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 2:
                internal::gemm_i32_2_0_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 3:
                internal::gemm_i32_2_0_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 4:
                internal::gemm_i32_2_0_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 5:
                internal::gemm_i32_2_0_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 6:
                internal::gemm_i32_2_0_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 7:
                internal::gemm_i32_2_0_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_2_1_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 1:
                internal::gemm_i32_2_1_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 2:
                internal::gemm_i32_2_1_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 3:
                internal::gemm_i32_2_1_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 4:
                internal::gemm_i32_2_1_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 5:
                internal::gemm_i32_2_1_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 6:
                internal::gemm_i32_2_1_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 7:
                internal::gemm_i32_2_1_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_i32_2_2_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 1:
                internal::gemm_i32_2_2_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 2:
                internal::gemm_i32_2_2_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 3:
                internal::gemm_i32_2_2_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 4:
                internal::gemm_i32_2_2_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 5:
                internal::gemm_i32_2_2_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 6:
                internal::gemm_i32_2_2_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
              case 7:
                internal::gemm_i32_2_2_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                         rhs_offset, result, result_stride);
                break;
            }
            break;
        }
        break;
    }
  }
}

void gemm_i32(std::uint8_t* scratch, const std::uint8_t* lhs,
              const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
              std::int32_t k, std::int32_t lhs_offset, std::int32_t rhs_offset,
              std::int32_t* result) {
  gemm_i32_strided(scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset, result,
                   n);
}

void gemm_f_strided(std::uint8_t* scratch, const std::uint8_t* lhs,
                    const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
                    std::int32_t k, std::int32_t lhs_offset,
                    std::int32_t rhs_offset, float result_scale, float* result,
                    std::int32_t result_stride) {
  const bool lhs_aligned = ((reinterpret_cast<std::uintptr_t>(lhs) % 8) == 0);
  const bool rhs_aligned = ((reinterpret_cast<std::uintptr_t>(rhs) % 8) == 0);
  const bool k_aligned = ((k % 8) == 0);
  const bool aligned = lhs_aligned && rhs_aligned && k_aligned;
  if (aligned) {
    switch (m % 3) {
      case 0:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_f_0_0_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 1:
                internal::gemm_f_0_0_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 2:
                internal::gemm_f_0_0_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 3:
                internal::gemm_f_0_0_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 4:
                internal::gemm_f_0_0_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 5:
                internal::gemm_f_0_0_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 6:
                internal::gemm_f_0_0_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 7:
                internal::gemm_f_0_0_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_f_0_1_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 1:
                internal::gemm_f_0_1_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 2:
                internal::gemm_f_0_1_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 3:
                internal::gemm_f_0_1_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 4:
                internal::gemm_f_0_1_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 5:
                internal::gemm_f_0_1_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 6:
                internal::gemm_f_0_1_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 7:
                internal::gemm_f_0_1_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_f_0_2_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 1:
                internal::gemm_f_0_2_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 2:
                internal::gemm_f_0_2_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 3:
                internal::gemm_f_0_2_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 4:
                internal::gemm_f_0_2_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 5:
                internal::gemm_f_0_2_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 6:
                internal::gemm_f_0_2_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 7:
                internal::gemm_f_0_2_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
            }
            break;
        }
        break;
      case 1:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_f_1_0_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 1:
                internal::gemm_f_1_0_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 2:
                internal::gemm_f_1_0_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 3:
                internal::gemm_f_1_0_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 4:
                internal::gemm_f_1_0_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 5:
                internal::gemm_f_1_0_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 6:
                internal::gemm_f_1_0_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 7:
                internal::gemm_f_1_0_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_f_1_1_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 1:
                internal::gemm_f_1_1_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 2:
                internal::gemm_f_1_1_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 3:
                internal::gemm_f_1_1_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 4:
                internal::gemm_f_1_1_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 5:
                internal::gemm_f_1_1_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 6:
                internal::gemm_f_1_1_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 7:
                internal::gemm_f_1_1_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_f_1_2_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 1:
                internal::gemm_f_1_2_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 2:
                internal::gemm_f_1_2_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 3:
                internal::gemm_f_1_2_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 4:
                internal::gemm_f_1_2_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 5:
                internal::gemm_f_1_2_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 6:
                internal::gemm_f_1_2_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 7:
                internal::gemm_f_1_2_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
            }
            break;
        }
        break;
      case 2:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_f_2_0_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 1:
                internal::gemm_f_2_0_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 2:
                internal::gemm_f_2_0_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 3:
                internal::gemm_f_2_0_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 4:
                internal::gemm_f_2_0_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 5:
                internal::gemm_f_2_0_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 6:
                internal::gemm_f_2_0_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 7:
                internal::gemm_f_2_0_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_f_2_1_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 1:
                internal::gemm_f_2_1_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 2:
                internal::gemm_f_2_1_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 3:
                internal::gemm_f_2_1_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 4:
                internal::gemm_f_2_1_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 5:
                internal::gemm_f_2_1_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 6:
                internal::gemm_f_2_1_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 7:
                internal::gemm_f_2_1_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_f_2_2_0_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 1:
                internal::gemm_f_2_2_1_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 2:
                internal::gemm_f_2_2_2_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 3:
                internal::gemm_f_2_2_3_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 4:
                internal::gemm_f_2_2_4_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 5:
                internal::gemm_f_2_2_5_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 6:
                internal::gemm_f_2_2_6_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
              case 7:
                internal::gemm_f_2_2_7_aligned(
                    scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                    result_scale, result, result_stride);
                break;
            }
            break;
        }
        break;
    }
  } else {
    switch (m % 3) {
      case 0:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_f_0_0_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 1:
                internal::gemm_f_0_0_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 2:
                internal::gemm_f_0_0_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 3:
                internal::gemm_f_0_0_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 4:
                internal::gemm_f_0_0_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 5:
                internal::gemm_f_0_0_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 6:
                internal::gemm_f_0_0_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 7:
                internal::gemm_f_0_0_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_f_0_1_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 1:
                internal::gemm_f_0_1_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 2:
                internal::gemm_f_0_1_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 3:
                internal::gemm_f_0_1_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 4:
                internal::gemm_f_0_1_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 5:
                internal::gemm_f_0_1_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 6:
                internal::gemm_f_0_1_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 7:
                internal::gemm_f_0_1_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_f_0_2_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 1:
                internal::gemm_f_0_2_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 2:
                internal::gemm_f_0_2_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 3:
                internal::gemm_f_0_2_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 4:
                internal::gemm_f_0_2_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 5:
                internal::gemm_f_0_2_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 6:
                internal::gemm_f_0_2_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 7:
                internal::gemm_f_0_2_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
            }
            break;
        }
        break;
      case 1:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_f_1_0_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 1:
                internal::gemm_f_1_0_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 2:
                internal::gemm_f_1_0_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 3:
                internal::gemm_f_1_0_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 4:
                internal::gemm_f_1_0_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 5:
                internal::gemm_f_1_0_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 6:
                internal::gemm_f_1_0_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 7:
                internal::gemm_f_1_0_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_f_1_1_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 1:
                internal::gemm_f_1_1_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 2:
                internal::gemm_f_1_1_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 3:
                internal::gemm_f_1_1_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 4:
                internal::gemm_f_1_1_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 5:
                internal::gemm_f_1_1_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 6:
                internal::gemm_f_1_1_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 7:
                internal::gemm_f_1_1_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_f_1_2_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 1:
                internal::gemm_f_1_2_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 2:
                internal::gemm_f_1_2_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 3:
                internal::gemm_f_1_2_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 4:
                internal::gemm_f_1_2_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 5:
                internal::gemm_f_1_2_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 6:
                internal::gemm_f_1_2_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 7:
                internal::gemm_f_1_2_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
            }
            break;
        }
        break;
      case 2:
        switch (n % 3) {
          case 0:
            switch (k % 8) {
              case 0:
                internal::gemm_f_2_0_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 1:
                internal::gemm_f_2_0_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 2:
                internal::gemm_f_2_0_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 3:
                internal::gemm_f_2_0_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 4:
                internal::gemm_f_2_0_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 5:
                internal::gemm_f_2_0_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 6:
                internal::gemm_f_2_0_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 7:
                internal::gemm_f_2_0_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
            }
            break;
          case 1:
            switch (k % 8) {
              case 0:
                internal::gemm_f_2_1_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 1:
                internal::gemm_f_2_1_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 2:
                internal::gemm_f_2_1_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 3:
                internal::gemm_f_2_1_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 4:
                internal::gemm_f_2_1_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 5:
                internal::gemm_f_2_1_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 6:
                internal::gemm_f_2_1_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 7:
                internal::gemm_f_2_1_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
            }
            break;
          case 2:
            switch (k % 8) {
              case 0:
                internal::gemm_f_2_2_0(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 1:
                internal::gemm_f_2_2_1(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 2:
                internal::gemm_f_2_2_2(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 3:
                internal::gemm_f_2_2_3(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 4:
                internal::gemm_f_2_2_4(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 5:
                internal::gemm_f_2_2_5(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 6:
                internal::gemm_f_2_2_6(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
              case 7:
                internal::gemm_f_2_2_7(scratch, lhs, rhs, m, n, k, lhs_offset,
                                       rhs_offset, result_scale, result,
                                       result_stride);
                break;
            }
            break;
        }
        break;
    }
  }
}

void gemm_f(std::uint8_t* scratch, const std::uint8_t* lhs,
            const std::uint8_t* rhs, std::int32_t m, std::int32_t n,
            std::int32_t k, std::int32_t lhs_offset, std::int32_t rhs_offset,
            float result_scale, float* result) {
  gemm_f_strided(scratch, lhs, rhs, m, n, k, lhs_offset, rhs_offset,
                 result_scale, result, n);
}

void gemv_q8(std::uint8_t* scratch, const std::uint8_t* lhs,
             const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
             std::int32_t lhs_offset, std::int32_t rhs_offset,
             std::int32_t result_offset, std::int32_t multiplicative_offset,
             std::int32_t shift, std::uint8_t* result) {
  const bool lhs_aligned = ((reinterpret_cast<std::uintptr_t>(lhs) % 8) == 0);
  const bool rhs_aligned = ((reinterpret_cast<std::uintptr_t>(rhs) % 8) == 0);
  const bool k_aligned = ((k % 8) == 0);
  const bool result_aligned =
      ((reinterpret_cast<std::uintptr_t>(result) % 8) == 0);
  const bool aligned =
      lhs_aligned && rhs_aligned && result_aligned && k_aligned;
  if (aligned) {
    switch (n % 8) {
      case 0:
        switch (k % 8) {
          case 0:
            internal::gemv_q8_0_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 1:
            internal::gemv_q8_0_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 2:
            internal::gemv_q8_0_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 3:
            internal::gemv_q8_0_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 4:
            internal::gemv_q8_0_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 5:
            internal::gemv_q8_0_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 6:
            internal::gemv_q8_0_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 7:
            internal::gemv_q8_0_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
        }
        break;
      case 1:
        switch (k % 8) {
          case 0:
            internal::gemv_q8_1_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 1:
            internal::gemv_q8_1_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 2:
            internal::gemv_q8_1_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 3:
            internal::gemv_q8_1_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 4:
            internal::gemv_q8_1_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 5:
            internal::gemv_q8_1_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 6:
            internal::gemv_q8_1_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 7:
            internal::gemv_q8_1_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
        }
        break;
      case 2:
        switch (k % 8) {
          case 0:
            internal::gemv_q8_2_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 1:
            internal::gemv_q8_2_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 2:
            internal::gemv_q8_2_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 3:
            internal::gemv_q8_2_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 4:
            internal::gemv_q8_2_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 5:
            internal::gemv_q8_2_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 6:
            internal::gemv_q8_2_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 7:
            internal::gemv_q8_2_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
        }
        break;
      case 3:
        switch (k % 8) {
          case 0:
            internal::gemv_q8_3_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 1:
            internal::gemv_q8_3_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 2:
            internal::gemv_q8_3_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 3:
            internal::gemv_q8_3_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 4:
            internal::gemv_q8_3_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 5:
            internal::gemv_q8_3_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 6:
            internal::gemv_q8_3_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 7:
            internal::gemv_q8_3_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
        }
        break;
      case 4:
        switch (k % 8) {
          case 0:
            internal::gemv_q8_4_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 1:
            internal::gemv_q8_4_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 2:
            internal::gemv_q8_4_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 3:
            internal::gemv_q8_4_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 4:
            internal::gemv_q8_4_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 5:
            internal::gemv_q8_4_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 6:
            internal::gemv_q8_4_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 7:
            internal::gemv_q8_4_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
        }
        break;
      case 5:
        switch (k % 8) {
          case 0:
            internal::gemv_q8_5_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 1:
            internal::gemv_q8_5_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 2:
            internal::gemv_q8_5_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 3:
            internal::gemv_q8_5_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 4:
            internal::gemv_q8_5_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 5:
            internal::gemv_q8_5_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 6:
            internal::gemv_q8_5_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 7:
            internal::gemv_q8_5_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
        }
        break;
      case 6:
        switch (k % 8) {
          case 0:
            internal::gemv_q8_6_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 1:
            internal::gemv_q8_6_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 2:
            internal::gemv_q8_6_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 3:
            internal::gemv_q8_6_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 4:
            internal::gemv_q8_6_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 5:
            internal::gemv_q8_6_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 6:
            internal::gemv_q8_6_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 7:
            internal::gemv_q8_6_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
        }
        break;
      case 7:
        switch (k % 8) {
          case 0:
            internal::gemv_q8_7_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 1:
            internal::gemv_q8_7_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 2:
            internal::gemv_q8_7_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 3:
            internal::gemv_q8_7_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 4:
            internal::gemv_q8_7_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 5:
            internal::gemv_q8_7_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 6:
            internal::gemv_q8_7_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
          case 7:
            internal::gemv_q8_7_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                          rhs_offset, result_offset,
                                          multiplicative_offset, shift, result);
            break;
        }
        break;
    }
  } else {
    switch (n % 8) {
      case 0:
        switch (k % 8) {
          case 0:
            internal::gemv_q8_0_0(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 1:
            internal::gemv_q8_0_1(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 2:
            internal::gemv_q8_0_2(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 3:
            internal::gemv_q8_0_3(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 4:
            internal::gemv_q8_0_4(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 5:
            internal::gemv_q8_0_5(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 6:
            internal::gemv_q8_0_6(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 7:
            internal::gemv_q8_0_7(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
        }
        break;
      case 1:
        switch (k % 8) {
          case 0:
            internal::gemv_q8_1_0(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 1:
            internal::gemv_q8_1_1(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 2:
            internal::gemv_q8_1_2(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 3:
            internal::gemv_q8_1_3(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 4:
            internal::gemv_q8_1_4(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 5:
            internal::gemv_q8_1_5(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 6:
            internal::gemv_q8_1_6(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 7:
            internal::gemv_q8_1_7(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
        }
        break;
      case 2:
        switch (k % 8) {
          case 0:
            internal::gemv_q8_2_0(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 1:
            internal::gemv_q8_2_1(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 2:
            internal::gemv_q8_2_2(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 3:
            internal::gemv_q8_2_3(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 4:
            internal::gemv_q8_2_4(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 5:
            internal::gemv_q8_2_5(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 6:
            internal::gemv_q8_2_6(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 7:
            internal::gemv_q8_2_7(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
        }
        break;
      case 3:
        switch (k % 8) {
          case 0:
            internal::gemv_q8_3_0(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 1:
            internal::gemv_q8_3_1(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 2:
            internal::gemv_q8_3_2(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 3:
            internal::gemv_q8_3_3(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 4:
            internal::gemv_q8_3_4(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 5:
            internal::gemv_q8_3_5(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 6:
            internal::gemv_q8_3_6(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 7:
            internal::gemv_q8_3_7(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
        }
        break;
      case 4:
        switch (k % 8) {
          case 0:
            internal::gemv_q8_4_0(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 1:
            internal::gemv_q8_4_1(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 2:
            internal::gemv_q8_4_2(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 3:
            internal::gemv_q8_4_3(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 4:
            internal::gemv_q8_4_4(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 5:
            internal::gemv_q8_4_5(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 6:
            internal::gemv_q8_4_6(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 7:
            internal::gemv_q8_4_7(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
        }
        break;
      case 5:
        switch (k % 8) {
          case 0:
            internal::gemv_q8_5_0(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 1:
            internal::gemv_q8_5_1(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 2:
            internal::gemv_q8_5_2(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 3:
            internal::gemv_q8_5_3(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 4:
            internal::gemv_q8_5_4(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 5:
            internal::gemv_q8_5_5(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 6:
            internal::gemv_q8_5_6(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 7:
            internal::gemv_q8_5_7(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
        }
        break;
      case 6:
        switch (k % 8) {
          case 0:
            internal::gemv_q8_6_0(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 1:
            internal::gemv_q8_6_1(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 2:
            internal::gemv_q8_6_2(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 3:
            internal::gemv_q8_6_3(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 4:
            internal::gemv_q8_6_4(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 5:
            internal::gemv_q8_6_5(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 6:
            internal::gemv_q8_6_6(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 7:
            internal::gemv_q8_6_7(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
        }
        break;
      case 7:
        switch (k % 8) {
          case 0:
            internal::gemv_q8_7_0(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 1:
            internal::gemv_q8_7_1(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 2:
            internal::gemv_q8_7_2(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 3:
            internal::gemv_q8_7_3(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 4:
            internal::gemv_q8_7_4(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 5:
            internal::gemv_q8_7_5(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 6:
            internal::gemv_q8_7_6(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
          case 7:
            internal::gemv_q8_7_7(scratch, lhs, rhs, n, k, lhs_offset,
                                  rhs_offset, result_offset,
                                  multiplicative_offset, shift, result);
            break;
        }
        break;
    }
  }
}

void gemv_i32(std::uint8_t* scratch, const std::uint8_t* lhs,
              const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
              std::int32_t lhs_offset, std::int32_t rhs_offset,
              std::int32_t* result) {
  const bool lhs_aligned = ((reinterpret_cast<std::uintptr_t>(lhs) % 8) == 0);
  const bool rhs_aligned = ((reinterpret_cast<std::uintptr_t>(rhs) % 8) == 0);
  const bool k_aligned = ((k % 8) == 0);
  const bool aligned = lhs_aligned && rhs_aligned && k_aligned;
  if (aligned) {
    switch (n % 8) {
      case 0:
        switch (k % 8) {
          case 0:
            internal::gemv_i32_0_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 1:
            internal::gemv_i32_0_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 2:
            internal::gemv_i32_0_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 3:
            internal::gemv_i32_0_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 4:
            internal::gemv_i32_0_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 5:
            internal::gemv_i32_0_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 6:
            internal::gemv_i32_0_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 7:
            internal::gemv_i32_0_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
        }
        break;
      case 1:
        switch (k % 8) {
          case 0:
            internal::gemv_i32_1_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 1:
            internal::gemv_i32_1_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 2:
            internal::gemv_i32_1_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 3:
            internal::gemv_i32_1_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 4:
            internal::gemv_i32_1_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 5:
            internal::gemv_i32_1_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 6:
            internal::gemv_i32_1_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 7:
            internal::gemv_i32_1_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
        }
        break;
      case 2:
        switch (k % 8) {
          case 0:
            internal::gemv_i32_2_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 1:
            internal::gemv_i32_2_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 2:
            internal::gemv_i32_2_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 3:
            internal::gemv_i32_2_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 4:
            internal::gemv_i32_2_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 5:
            internal::gemv_i32_2_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 6:
            internal::gemv_i32_2_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 7:
            internal::gemv_i32_2_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
        }
        break;
      case 3:
        switch (k % 8) {
          case 0:
            internal::gemv_i32_3_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 1:
            internal::gemv_i32_3_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 2:
            internal::gemv_i32_3_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 3:
            internal::gemv_i32_3_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 4:
            internal::gemv_i32_3_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 5:
            internal::gemv_i32_3_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 6:
            internal::gemv_i32_3_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 7:
            internal::gemv_i32_3_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
        }
        break;
      case 4:
        switch (k % 8) {
          case 0:
            internal::gemv_i32_4_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 1:
            internal::gemv_i32_4_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 2:
            internal::gemv_i32_4_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 3:
            internal::gemv_i32_4_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 4:
            internal::gemv_i32_4_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 5:
            internal::gemv_i32_4_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 6:
            internal::gemv_i32_4_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 7:
            internal::gemv_i32_4_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
        }
        break;
      case 5:
        switch (k % 8) {
          case 0:
            internal::gemv_i32_5_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 1:
            internal::gemv_i32_5_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 2:
            internal::gemv_i32_5_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 3:
            internal::gemv_i32_5_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 4:
            internal::gemv_i32_5_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 5:
            internal::gemv_i32_5_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 6:
            internal::gemv_i32_5_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 7:
            internal::gemv_i32_5_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
        }
        break;
      case 6:
        switch (k % 8) {
          case 0:
            internal::gemv_i32_6_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 1:
            internal::gemv_i32_6_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 2:
            internal::gemv_i32_6_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 3:
            internal::gemv_i32_6_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 4:
            internal::gemv_i32_6_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 5:
            internal::gemv_i32_6_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 6:
            internal::gemv_i32_6_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 7:
            internal::gemv_i32_6_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
        }
        break;
      case 7:
        switch (k % 8) {
          case 0:
            internal::gemv_i32_7_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 1:
            internal::gemv_i32_7_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 2:
            internal::gemv_i32_7_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 3:
            internal::gemv_i32_7_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 4:
            internal::gemv_i32_7_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 5:
            internal::gemv_i32_7_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 6:
            internal::gemv_i32_7_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
          case 7:
            internal::gemv_i32_7_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                           rhs_offset, result);
            break;
        }
        break;
    }
  } else {
    switch (n % 8) {
      case 0:
        switch (k % 8) {
          case 0:
            internal::gemv_i32_0_0(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 1:
            internal::gemv_i32_0_1(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 2:
            internal::gemv_i32_0_2(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 3:
            internal::gemv_i32_0_3(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 4:
            internal::gemv_i32_0_4(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 5:
            internal::gemv_i32_0_5(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 6:
            internal::gemv_i32_0_6(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 7:
            internal::gemv_i32_0_7(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
        }
        break;
      case 1:
        switch (k % 8) {
          case 0:
            internal::gemv_i32_1_0(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 1:
            internal::gemv_i32_1_1(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 2:
            internal::gemv_i32_1_2(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 3:
            internal::gemv_i32_1_3(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 4:
            internal::gemv_i32_1_4(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 5:
            internal::gemv_i32_1_5(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 6:
            internal::gemv_i32_1_6(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 7:
            internal::gemv_i32_1_7(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
        }
        break;
      case 2:
        switch (k % 8) {
          case 0:
            internal::gemv_i32_2_0(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 1:
            internal::gemv_i32_2_1(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 2:
            internal::gemv_i32_2_2(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 3:
            internal::gemv_i32_2_3(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 4:
            internal::gemv_i32_2_4(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 5:
            internal::gemv_i32_2_5(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 6:
            internal::gemv_i32_2_6(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 7:
            internal::gemv_i32_2_7(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
        }
        break;
      case 3:
        switch (k % 8) {
          case 0:
            internal::gemv_i32_3_0(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 1:
            internal::gemv_i32_3_1(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 2:
            internal::gemv_i32_3_2(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 3:
            internal::gemv_i32_3_3(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 4:
            internal::gemv_i32_3_4(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 5:
            internal::gemv_i32_3_5(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 6:
            internal::gemv_i32_3_6(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 7:
            internal::gemv_i32_3_7(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
        }
        break;
      case 4:
        switch (k % 8) {
          case 0:
            internal::gemv_i32_4_0(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 1:
            internal::gemv_i32_4_1(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 2:
            internal::gemv_i32_4_2(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 3:
            internal::gemv_i32_4_3(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 4:
            internal::gemv_i32_4_4(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 5:
            internal::gemv_i32_4_5(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 6:
            internal::gemv_i32_4_6(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 7:
            internal::gemv_i32_4_7(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
        }
        break;
      case 5:
        switch (k % 8) {
          case 0:
            internal::gemv_i32_5_0(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 1:
            internal::gemv_i32_5_1(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 2:
            internal::gemv_i32_5_2(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 3:
            internal::gemv_i32_5_3(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 4:
            internal::gemv_i32_5_4(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 5:
            internal::gemv_i32_5_5(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 6:
            internal::gemv_i32_5_6(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 7:
            internal::gemv_i32_5_7(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
        }
        break;
      case 6:
        switch (k % 8) {
          case 0:
            internal::gemv_i32_6_0(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 1:
            internal::gemv_i32_6_1(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 2:
            internal::gemv_i32_6_2(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 3:
            internal::gemv_i32_6_3(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 4:
            internal::gemv_i32_6_4(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 5:
            internal::gemv_i32_6_5(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 6:
            internal::gemv_i32_6_6(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 7:
            internal::gemv_i32_6_7(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
        }
        break;
      case 7:
        switch (k % 8) {
          case 0:
            internal::gemv_i32_7_0(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 1:
            internal::gemv_i32_7_1(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 2:
            internal::gemv_i32_7_2(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 3:
            internal::gemv_i32_7_3(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 4:
            internal::gemv_i32_7_4(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 5:
            internal::gemv_i32_7_5(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 6:
            internal::gemv_i32_7_6(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
          case 7:
            internal::gemv_i32_7_7(scratch, lhs, rhs, n, k, lhs_offset,
                                   rhs_offset, result);
            break;
        }
        break;
    }
  }
}

void gemv_f(std::uint8_t* scratch, const std::uint8_t* lhs,
            const std::uint8_t* rhs, std::int32_t n, std::int32_t k,
            std::int32_t lhs_offset, std::int32_t rhs_offset,
            float result_scale, float* result) {
  const bool lhs_aligned = ((reinterpret_cast<std::uintptr_t>(lhs) % 8) == 0);
  const bool rhs_aligned = ((reinterpret_cast<std::uintptr_t>(rhs) % 8) == 0);
  const bool k_aligned = ((k % 8) == 0);
  const bool aligned = lhs_aligned && rhs_aligned && k_aligned;
  if (aligned) {
    switch (n % 8) {
      case 0:
        switch (k % 8) {
          case 0:
            internal::gemv_f_0_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 1:
            internal::gemv_f_0_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 2:
            internal::gemv_f_0_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 3:
            internal::gemv_f_0_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 4:
            internal::gemv_f_0_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 5:
            internal::gemv_f_0_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 6:
            internal::gemv_f_0_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 7:
            internal::gemv_f_0_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
        }
        break;
      case 1:
        switch (k % 8) {
          case 0:
            internal::gemv_f_1_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 1:
            internal::gemv_f_1_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 2:
            internal::gemv_f_1_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 3:
            internal::gemv_f_1_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 4:
            internal::gemv_f_1_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 5:
            internal::gemv_f_1_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 6:
            internal::gemv_f_1_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 7:
            internal::gemv_f_1_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
        }
        break;
      case 2:
        switch (k % 8) {
          case 0:
            internal::gemv_f_2_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 1:
            internal::gemv_f_2_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 2:
            internal::gemv_f_2_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 3:
            internal::gemv_f_2_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 4:
            internal::gemv_f_2_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 5:
            internal::gemv_f_2_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 6:
            internal::gemv_f_2_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 7:
            internal::gemv_f_2_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
        }
        break;
      case 3:
        switch (k % 8) {
          case 0:
            internal::gemv_f_3_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 1:
            internal::gemv_f_3_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 2:
            internal::gemv_f_3_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 3:
            internal::gemv_f_3_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 4:
            internal::gemv_f_3_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 5:
            internal::gemv_f_3_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 6:
            internal::gemv_f_3_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 7:
            internal::gemv_f_3_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
        }
        break;
      case 4:
        switch (k % 8) {
          case 0:
            internal::gemv_f_4_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 1:
            internal::gemv_f_4_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 2:
            internal::gemv_f_4_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 3:
            internal::gemv_f_4_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 4:
            internal::gemv_f_4_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 5:
            internal::gemv_f_4_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 6:
            internal::gemv_f_4_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 7:
            internal::gemv_f_4_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
        }
        break;
      case 5:
        switch (k % 8) {
          case 0:
            internal::gemv_f_5_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 1:
            internal::gemv_f_5_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 2:
            internal::gemv_f_5_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 3:
            internal::gemv_f_5_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 4:
            internal::gemv_f_5_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 5:
            internal::gemv_f_5_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 6:
            internal::gemv_f_5_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 7:
            internal::gemv_f_5_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
        }
        break;
      case 6:
        switch (k % 8) {
          case 0:
            internal::gemv_f_6_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 1:
            internal::gemv_f_6_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 2:
            internal::gemv_f_6_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 3:
            internal::gemv_f_6_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 4:
            internal::gemv_f_6_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 5:
            internal::gemv_f_6_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 6:
            internal::gemv_f_6_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 7:
            internal::gemv_f_6_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
        }
        break;
      case 7:
        switch (k % 8) {
          case 0:
            internal::gemv_f_7_0_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 1:
            internal::gemv_f_7_1_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 2:
            internal::gemv_f_7_2_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 3:
            internal::gemv_f_7_3_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 4:
            internal::gemv_f_7_4_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 5:
            internal::gemv_f_7_5_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 6:
            internal::gemv_f_7_6_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
          case 7:
            internal::gemv_f_7_7_aligned(scratch, lhs, rhs, n, k, lhs_offset,
                                         rhs_offset, result_scale, result);
            break;
        }
        break;
    }
  } else {
    switch (n % 8) {
      case 0:
        switch (k % 8) {
          case 0:
            internal::gemv_f_0_0(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 1:
            internal::gemv_f_0_1(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 2:
            internal::gemv_f_0_2(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 3:
            internal::gemv_f_0_3(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 4:
            internal::gemv_f_0_4(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 5:
            internal::gemv_f_0_5(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 6:
            internal::gemv_f_0_6(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 7:
            internal::gemv_f_0_7(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
        }
        break;
      case 1:
        switch (k % 8) {
          case 0:
            internal::gemv_f_1_0(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 1:
            internal::gemv_f_1_1(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 2:
            internal::gemv_f_1_2(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 3:
            internal::gemv_f_1_3(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 4:
            internal::gemv_f_1_4(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 5:
            internal::gemv_f_1_5(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 6:
            internal::gemv_f_1_6(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 7:
            internal::gemv_f_1_7(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
        }
        break;
      case 2:
        switch (k % 8) {
          case 0:
            internal::gemv_f_2_0(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 1:
            internal::gemv_f_2_1(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 2:
            internal::gemv_f_2_2(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 3:
            internal::gemv_f_2_3(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 4:
            internal::gemv_f_2_4(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 5:
            internal::gemv_f_2_5(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 6:
            internal::gemv_f_2_6(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 7:
            internal::gemv_f_2_7(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
        }
        break;
      case 3:
        switch (k % 8) {
          case 0:
            internal::gemv_f_3_0(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 1:
            internal::gemv_f_3_1(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 2:
            internal::gemv_f_3_2(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 3:
            internal::gemv_f_3_3(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 4:
            internal::gemv_f_3_4(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 5:
            internal::gemv_f_3_5(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 6:
            internal::gemv_f_3_6(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 7:
            internal::gemv_f_3_7(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
        }
        break;
      case 4:
        switch (k % 8) {
          case 0:
            internal::gemv_f_4_0(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 1:
            internal::gemv_f_4_1(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 2:
            internal::gemv_f_4_2(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 3:
            internal::gemv_f_4_3(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 4:
            internal::gemv_f_4_4(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 5:
            internal::gemv_f_4_5(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 6:
            internal::gemv_f_4_6(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 7:
            internal::gemv_f_4_7(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
        }
        break;
      case 5:
        switch (k % 8) {
          case 0:
            internal::gemv_f_5_0(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 1:
            internal::gemv_f_5_1(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 2:
            internal::gemv_f_5_2(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 3:
            internal::gemv_f_5_3(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 4:
            internal::gemv_f_5_4(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 5:
            internal::gemv_f_5_5(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 6:
            internal::gemv_f_5_6(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 7:
            internal::gemv_f_5_7(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
        }
        break;
      case 6:
        switch (k % 8) {
          case 0:
            internal::gemv_f_6_0(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 1:
            internal::gemv_f_6_1(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 2:
            internal::gemv_f_6_2(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 3:
            internal::gemv_f_6_3(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 4:
            internal::gemv_f_6_4(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 5:
            internal::gemv_f_6_5(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 6:
            internal::gemv_f_6_6(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 7:
            internal::gemv_f_6_7(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
        }
        break;
      case 7:
        switch (k % 8) {
          case 0:
            internal::gemv_f_7_0(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 1:
            internal::gemv_f_7_1(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 2:
            internal::gemv_f_7_2(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 3:
            internal::gemv_f_7_3(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 4:
            internal::gemv_f_7_4(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 5:
            internal::gemv_f_7_5(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 6:
            internal::gemv_f_7_6(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
          case 7:
            internal::gemv_f_7_7(scratch, lhs, rhs, n, k, lhs_offset,
                                 rhs_offset, result_scale, result);
            break;
        }
        break;
    }
  }
}


}  // namespace meta
}  // namespace gemmlowp

#else
#warning "Meta gemm fast-path requires GEMMLOWP_NEON_(32|64)!"
#endif

#endif  // GEMMLOWP_META_SINGLE_THREAD_GEMM_H_
