/*
 * Copyright (c) 2022, 2024, Intel Corporation. All rights reserved.
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 *
 */

#include "macroAssembler_x86.hpp"
#include "stubGenerator_x86_64.hpp"

#define __ _masm->

// References:
//  - (Normative) RFC7539 - ChaCha20 and Poly1305 for IETF Protocols
//  - M. Goll and S. Gueron, "Vectorization of Poly1305 Message Authentication Code"
//  - "The design of Poly1305" https://loup-vaillant.fr/tutorials/poly1305-design

// Explanation for the 'well known' modular arithmetic optimization, reduction by pseudo-Mersene prime 2^130-5:
//
// Reduction by 2^130-5 can be expressed as follows:
//    ( ax2^130 + b ) mod 2^130-5     //i.e. number split along the 130-bit boundary
//                                 = ( ax2^130 - 5xa + 5xa + b ) mod 2^130-5
//                                 = ( ax(2^130 - 5) + 5xa + b ) mod 2^130-5 // i.e. adding multiples of modulus is a noop
//                                 = ( 5xa + b ) mod 2^130-5
// QED: shows mathematically the well known algorithm of 'split the number down the middle, multiply upper and add'
// This is particularly useful to understand when combining with 'odd-sized' limbs that might cause misallignment
//

// Pseudocode for this file (in general):
//    * used for poly1305_multiply_scalar
//    x used for poly1305_multiply8_avx512
//    lower-case variables are scalar numbers in 3x44-bit limbs (in gprs)
//    upper-case variables are 8-element vector numbers in 3x44-bit limbs (in zmm registers)
//    [ ] used to denote vector numbers (with their elements)

// Constant Pool:
ATTRIBUTE_ALIGNED(64) static const uint64_t POLY1305_PAD_MSG[] = {
  0x0000010000000000, 0x0000010000000000,
  0x0000010000000000, 0x0000010000000000,
  0x0000010000000000, 0x0000010000000000,
  0x0000010000000000, 0x0000010000000000,
};
static address poly1305_pad_msg() {
  return (address)POLY1305_PAD_MSG;
}

ATTRIBUTE_ALIGNED(64) static const uint64_t POLY1305_MASK42[] = {
  0x000003ffffffffff, 0x000003ffffffffff,
  0x000003ffffffffff, 0x000003ffffffffff,
  0x000003ffffffffff, 0x000003ffffffffff,
  0x000003ffffffffff, 0x000003ffffffffff
};
static address poly1305_mask42() {
  return (address)POLY1305_MASK42;
}

ATTRIBUTE_ALIGNED(64) static const uint64_t POLY1305_MASK44[] = {
  0x00000fffffffffff, 0x00000fffffffffff,
  0x00000fffffffffff, 0x00000fffffffffff,
  0x00000fffffffffff, 0x00000fffffffffff,
  0x00000fffffffffff, 0x00000fffffffffff,
};
static address poly1305_mask44() {
  return (address)POLY1305_MASK44;
}

// Compute product for 8 16-byte message blocks,
// i.e. For each block, compute [a2 a1 a0] = [a2 a1 a0] x [r2 r1 r0]
//
// Each block/number is represented by 3 44-bit limb digits, start with multiplication
//
//      a2       a1       a0
// x    r2       r1       r0
// ----------------------------------
//     a2xr0    a1xr0    a0xr0
// +   a1xr1    a0xr1  5xa2xr1'     (r1' = r1<<2)
// +   a0xr2  5xa2xr2' 5xa1xr2'     (r2' = r2<<2)
// ----------------------------------
//        p2       p1       p0
//
// Then, propagate the carry (bits after bit 44) from lower limbs into higher limbs.
// Then, modular reduction from upper limb wrapped to lower limbs
//
// Math Note 1: 'carry propagation' from p2 to p0 involves multiplication by 5 (i.e. slightly modified modular reduction from above):
//    ( p2x2^88 ) mod 2^130-5
//                             = ( p2'x2^88 + p2''x2^130) mod 2^130-5 // Split on 130-bit boudary
//                             = ( p2'x2^88 + p2''x2^130 - 5xp2'' + 5xp2'') mod 2^130-5
//                             = ( p2'x2^88 + p2''x(2^130 - 5) + 5xp2'') mod 2^130-5 // i.e. adding multiples of modulus is a noop
//                             = ( p2'x2^88 + 5xp2'') mod 2^130-5
//
// Math Note 2: R1P = 4*5*R1 and R2P = 4*5*R2; This precomputation allows simultaneous reduction and multiplication.
// This is not the standard 'multiply-upper-by-5', here is why the factor is 4*5 instead of 5.
// For example, partial product (a2xr2):
//    (a2x2^88)x(r2x2^88) mod 2^130-5
//                                    = (a2xr2 x 2^176) mod 2^130-5
//                                    = (a2xr2 x 2^46x2^130) mod 2^130-5
//                                    = (a2xr2x2^46 x 2^130- 5xa2xr2x2^46 + 5xa2xr2x2^46) mod 2^130-5
//                                    = (a2xr2x2^46 x (2^130- 5) + 5xa2xr2x2^46) mod 2^130-5 // i.e. adding multiples of modulus is a noop
//                                    = (5xa2xr2x2^46) mod 2^130-5
//                                    = (a2x5xr2x2^2 x 2^44) mod 2^130-5 // Align to limb boudary
//                                    = (a2x[5xr2x4] x 2^44) mod 2^130-5
//                                    = (a2xR2P x 2^44) mod 2^130-5 // i.e. R2P = 4*5*R2
//
void StubGenerator::poly1305_multiply8_avx512(
  const XMMRegister A0, const XMMRegister A1, const XMMRegister A2,
  const XMMRegister R0, const XMMRegister R1, const XMMRegister R2, const XMMRegister R1P, const XMMRegister R2P,
  const XMMRegister P0L, const XMMRegister P0H, const XMMRegister P1L, const XMMRegister P1H, const XMMRegister P2L, const XMMRegister P2H,
  const XMMRegister TMP, const Register rscratch)
{

  // Reset partial sums
  __ evpxorq(P0L, P0L, P0L, Assembler::AVX_512bit);
  __ evpxorq(P0H, P0H, P0H, Assembler::AVX_512bit);
  __ evpxorq(P1L, P1L, P1L, Assembler::AVX_512bit);
  __ evpxorq(P1H, P1H, P1H, Assembler::AVX_512bit);
  __ evpxorq(P2L, P2L, P2L, Assembler::AVX_512bit);
  __ evpxorq(P2H, P2H, P2H, Assembler::AVX_512bit);

  // Calculate partial products
  // p0 = a2xr1'
  // p1 = a2xr2'
  // p2 = a2xr0
  __ evpmadd52luq(P0L, A2, R1P, Assembler::AVX_512bit);
  __ evpmadd52huq(P0H, A2, R1P, Assembler::AVX_512bit);
  __ evpmadd52luq(P1L, A2, R2P, Assembler::AVX_512bit);
  __ evpmadd52huq(P1H, A2, R2P, Assembler::AVX_512bit);
  __ evpmadd52luq(P2L, A2, R0, Assembler::AVX_512bit);
  __ evpmadd52huq(P2H, A2, R0, Assembler::AVX_512bit);

  // p0 += a0xr0
  // p1 += a0xr1
  // p2 += a0xr2
  __ evpmadd52luq(P1L, A0, R1, Assembler::AVX_512bit);
  __ evpmadd52huq(P1H, A0, R1, Assembler::AVX_512bit);
  __ evpmadd52luq(P2L, A0, R2, Assembler::AVX_512bit);
  __ evpmadd52huq(P2H, A0, R2, Assembler::AVX_512bit);
  __ evpmadd52luq(P0L, A0, R0, Assembler::AVX_512bit);
  __ evpmadd52huq(P0H, A0, R0, Assembler::AVX_512bit);

  // p0 += a1xr2'
  // p1 += a1xr0
  // p2 += a1xr1
  __ evpmadd52luq(P0L, A1, R2P, Assembler::AVX_512bit);
  __ evpmadd52huq(P0H, A1, R2P, Assembler::AVX_512bit);
  __ evpmadd52luq(P1L, A1, R0, Assembler::AVX_512bit);
  __ evpmadd52huq(P1H, A1, R0, Assembler::AVX_512bit);
  __ evpmadd52luq(P2L, A1, R1, Assembler::AVX_512bit);
  __ evpmadd52huq(P2H, A1, R1, Assembler::AVX_512bit);

  // Carry propagation:
  // (Not quite aligned)                         | More mathematically correct:
  //         P2L   P1L   P0L                     |                 P2Lx2^88 + P1Lx2^44 + P0Lx2^0
  // + P2H   P1H   P0H                           |   + P2Hx2^140 + P1Hx2^96 + P0Hx2^52
  // ---------------------------                 |   -----------------------------------------------
  // = P2H    A2    A1    A0                     |   = P2Hx2^130 + A2x2^88 +   A1x2^44 +  A0x2^0
  //
  __ vpsrlq(TMP, P0L, 44, Assembler::AVX_512bit);
  __ evpandq(A0, P0L, ExternalAddress(poly1305_mask44()), Assembler::AVX_512bit, rscratch); // Clear top 20 bits

  __ vpsllq(P0H, P0H, 8, Assembler::AVX_512bit);
  __ vpaddq(P0H, P0H, TMP, Assembler::AVX_512bit);
  __ vpaddq(P1L, P1L, P0H, Assembler::AVX_512bit);
  __ evpandq(A1, P1L, ExternalAddress(poly1305_mask44()), Assembler::AVX_512bit, rscratch); // Clear top 20 bits

  __ vpsrlq(TMP, P1L, 44, Assembler::AVX_512bit);
  __ vpsllq(P1H, P1H, 8, Assembler::AVX_512bit);
  __ vpaddq(P1H, P1H, TMP, Assembler::AVX_512bit);
  __ vpaddq(P2L, P2L, P1H, Assembler::AVX_512bit);
  __ evpandq(A2, P2L, ExternalAddress(poly1305_mask42()), Assembler::AVX_512bit, rscratch); // Clear top 22 bits

  __ vpsrlq(TMP, P2L, 42, Assembler::AVX_512bit);
  __ vpsllq(P2H, P2H, 10, Assembler::AVX_512bit);
  __ vpaddq(P2H, P2H, TMP, Assembler::AVX_512bit);

  // Reduction: p2->a0->a1
  // Multiply by 5 the highest bits (p2 is above 130 bits)
  __ vpaddq(A0, A0, P2H, Assembler::AVX_512bit);
  __ vpsllq(P2H, P2H, 2, Assembler::AVX_512bit);
  __ vpaddq(A0, A0, P2H, Assembler::AVX_512bit);
  __ vpsrlq(TMP, A0, 44, Assembler::AVX_512bit);
  __ evpandq(A0, A0, ExternalAddress(poly1305_mask44()), Assembler::AVX_512bit, rscratch);
  __ vpaddq(A1, A1, TMP, Assembler::AVX_512bit);
}

// Compute product for a single 16-byte message blocks
// - Assumes that r = [r1 r0] is only 128 bits (not 130)
// - Input [a2 a1 a0]; when only128 is set, input is 128 bits (i.e. a2==0)
// - Output [a2 a1 a0] is at least 130 bits (i.e. a2 is used regardless of only128)
//
// Note 1: a2 here is only two bits so anything above is subject of reduction.
// Note 2: Constant c1 = 5xr1 = r1 + (r1 << 2) simplifies multiply with less operations
//
// Flow of the code below is as follows:
//
//          a2        a1        a0
//        x           r1        r0
//   -----------------------------
//       a2xr0     a1xr0     a0xr0
//   +             a0xr1
//   +           5xa2xr1   5xa1xr1
//   -----------------------------
//     [0|L2L] [L1H|L1L] [L0H|L0L]
//
//   Registers:  t2:t1     t0:a0
//
// Completing the multiply and adding (with carry) 3x128-bit limbs into
// 192-bits again (3x64-bits):
// a0 = L0L
// a1 = L0H + L1L
// t2 = L1H + L2L
void StubGenerator::poly1305_multiply_scalar(
  const Register a0, const Register a1, const Register a2,
  const Register r0, const Register r1, const Register c1, bool only128,
  const Register t0, const Register t1, const Register t2,
  const Register mulql, const Register mulqh)
{
  // mulq instruction requires/clobers rax, rdx (mulql, mulqh)

  // t2:t1 = (a0 * r1)
  __ movq(rax, r1);
  __ mulq(a0);
  __ movq(t1, rax);
  __ movq(t2, rdx);

  // t0:a0 = (a0 * r0)
  __ movq(rax, r0);
  __ mulq(a0);
  __ movq(a0, rax); // a0 not used in other operations
  __ movq(t0, rdx);

  // t2:t1 += (a1 * r0)
  __ movq(rax, r0);
  __ mulq(a1);
  __ addq(t1, rax);
  __ adcq(t2, rdx);

  // t0:a0 += (a1 * r1x5)
  __ movq(rax, c1);
  __ mulq(a1);
  __ addq(a0, rax);
  __ adcq(t0, rdx);

  // Note: a2 is clamped to 2-bits,
  //       r1/r0 is clamped to 60-bits,
  //       their product is less than 2^64.

  if (only128) { // Accumulator only 128 bits, i.e. a2 == 0
    // just move and add t0-t1 to a1
    __ movq(a1, t0);
    __ addq(a1, t1);
    __ adcq(t2, 0);
  } else {
    // t2:t1 += (a2 * r1x5)
    __ movq(a1, a2); // use a1 for a2
    __ imulq(a1, c1);
    __ addq(t1, a1);
    __ adcq(t2, 0);

    __ movq(a1, t0); // t0:a0 => a1:a0

    // t2:a1 += (a2 * r0):t1
    __ imulq(a2, r0);
    __ addq(a1, t1);
    __ adcq(t2, a2);
  }

  // At this point, 3 64-bit limbs are in t2:a1:a0
  // t2 can span over more than 2 bits so final partial reduction step is needed.
  //
  // Partial reduction (just to fit into 130 bits)
  //    a2 = t2 & 3
  //    k = (t2 & ~3) + (t2 >> 2)
  //         Y    x4  +  Y    x1
  //    a2:a1:a0 += k
  //
  // Result will be in a2:a1:a0
  __ movq(t0, t2);
  __ movl(a2, t2); // DWORD
  __ andq(t0, ~3);
  __ shrq(t2, 2);
  __ addq(t0, t2);
  __ andl(a2, 3); // DWORD

  // a2:a1:a0 += k (kept in t0)
  __ addq(a0, t0);
  __ adcq(a1, 0);
  __ adcl(a2, 0); // DWORD
}

// Convert array of 128-bit numbers in quadwords (in D0:D1) into 128-bit numbers across 44-bit limbs (in L0:L1:L2)
// Optionally pad all the numbers (i.e. add 2^128)
//
//         +-------------------------+-------------------------+
//  D0:D1  | h0 h1 g0 g1 f0 f1 e0 e1 | d0 d1 c0 c1 b0 b1 a0 a1 |
//         +-------------------------+-------------------------+
//         +-------------------------+
//  L2     | h2 d2 g2 c2 f2 b2 e2 a2 |
//         +-------------------------+
//         +-------------------------+
//  L1     | h1 d1 g1 c1 f1 b1 e1 a1 |
//         +-------------------------+
//         +-------------------------+
//  L0     | h0 d0 g0 c0 f0 b0 e0 a0 |
//         +-------------------------+
//
void StubGenerator::poly1305_limbs_avx512(
    const XMMRegister D0, const XMMRegister D1,
    const XMMRegister L0, const XMMRegister L1, const XMMRegister L2, bool padMSG,
    const XMMRegister TMP, const Register rscratch)
{
  // Interleave blocks of data
  __ evpunpckhqdq(TMP, D0, D1, Assembler::AVX_512bit);
  __ evpunpcklqdq(L0, D0, D1, Assembler::AVX_512bit);

  // Highest 42-bit limbs of new blocks
  __ vpsrlq(L2, TMP, 24, Assembler::AVX_512bit);
  if (padMSG) {
    __ evporq(L2, L2, ExternalAddress(poly1305_pad_msg()), Assembler::AVX_512bit, rscratch); // Add 2^128 to all 8 final qwords of the message
  }

  // Middle 44-bit limbs of new blocks
  __ vpsrlq(L1, L0, 44, Assembler::AVX_512bit);
  __ vpsllq(TMP, TMP, 20, Assembler::AVX_512bit);
  __ vpternlogq(L1, 0xA8, TMP, ExternalAddress(poly1305_mask44()), Assembler::AVX_512bit, rscratch); // (A OR B AND C)

  // Lowest 44-bit limbs of new blocks
  __ evpandq(L0, L0, ExternalAddress(poly1305_mask44()), Assembler::AVX_512bit, rscratch);
}

/**
 * Copy 5x26-bit (unreduced) limbs stored at Register limbs into  a2:a1:a0 (3x64-bit limbs)
 *
 * a2 is optional. When only128 is set, limbs are expected to fit into 128-bits (i.e. a1:a0 such as clamped R)
 */
void StubGenerator::poly1305_limbs(
    const Register limbs, const Register a0, const Register a1, const Register a2,
    const Register t0, const Register t1)
{
  __ movq(a0, Address(limbs, 0));
  __ movq(t0, Address(limbs, 8));
  __ shlq(t0, 26);
  __ addq(a0, t0);
  __ movq(t0, Address(limbs, 16));
  __ movq(t1, Address(limbs, 24));
  __ movq(a1, t0);
  __ shlq(t0, 52);
  __ shrq(a1, 12);
  __ shlq(t1, 14);
  __ addq(a0, t0);
  __ adcq(a1, t1);
  __ movq(t0, Address(limbs, 32));
  if (a2 != noreg) {
    __ movq(a2, t0);
    __ shrq(a2, 24);
  }
  __ shlq(t0, 40);
  __ addq(a1, t0);
  if (a2 != noreg) {
    __ adcq(a2, 0);

    // One round of reduction
    // Take bits above 130 in a2, multiply by 5 and add to a2:a1:a0
    __ movq(t0, a2);
    __ andq(t0, ~3);
    __ andq(a2, 3);
    __ movq(t1, t0);
    __ shrq(t1, 2);
    __ addq(t0, t1);

    __ addq(a0, t0);
    __ adcq(a1, 0);
    __ adcq(a2, 0);
  }
}

/**
 * Break 3x64-bit a2:a1:a0 limbs into 5x26-bit limbs and store out into 5 quadwords at address `limbs`
 */
void StubGenerator::poly1305_limbs_out(
    const Register a0, const Register a1, const Register a2,
    const Register limbs,
    const Register t0, const Register t1)
{
  // Extra round of reduction
  // Take bits above 130 in a2, multiply by 5 and add to a2:a1:a0
  __ movq(t0, a2);
  __ andq(t0, ~3);
  __ andq(a2, 3);
  __ movq(t1, t0);
  __ shrq(t1, 2);
  __ addq(t0, t1);

  __ addq(a0, t0);
  __ adcq(a1, 0);
  __ adcq(a2, 0);

  // Chop a2:a1:a0 into 26-bit limbs
  __ movl(t0, a0);
  __ andl(t0, 0x3ffffff);
  __ movq(Address(limbs, 0), t0);

  __ shrq(a0, 26);
  __ movl(t0, a0);
  __ andl(t0, 0x3ffffff);
  __ movq(Address(limbs, 8), t0);

  __ shrq(a0, 26); // 12 bits left in a0, concatenate 14 from a1
  __ movl(t0, a1);
  __ shll(t0, 12);
  __ addl(t0, a0);
  __ andl(t0, 0x3ffffff);
  __ movq(Address(limbs, 16), t0);

  __ shrq(a1, 14); // already used up 14 bits
  __ shlq(a2, 50); // a2 contains 2 bits when reduced, but $Element.limbs dont have to be fully reduced
  __ addq(a1, a2); // put remaining bits into a1

  __ movl(t0, a1);
  __ andl(t0, 0x3ffffff);
  __ movq(Address(limbs, 24), t0);

  __ shrq(a1, 26);
  __ movl(t0, a1);
  //andl(t0, 0x3ffffff); doesnt have to be fully reduced, leave remaining bit(s)
  __ movq(Address(limbs, 32), t0);
}

// This function consumes as many whole 16*16-byte blocks as available in input
// After execution, input and length will point at remaining (unprocessed) data
// and [a2 a1 a0] will contain the current accumulator value
//
// Math Note:
//    Main loop in this function multiplies each message block by r^16; And some glue before and after..
//    Proof (for brevity, split into 4 'rows' instead of 16):
//
//     hash = ((((m1*r + m2)*r + m3)*r ...  mn)*r
//          = m1*r^n + m2*r^(n-1) + ... +mn_1*r^2 + mn*r  // Horner's rule
//
//          = m1*r^n     + m4*r^(n-4) + m8*r^(n-8) ...    // split into 4 groups for brevity, same applies to 16 blocks
//          + m2*r^(n-1) + m5*r^(n-5) + m9*r^(n-9) ...
//          + m3*r^(n-2) + m6*r^(n-6) + m10*r^(n-10) ...
//          + m4*r^(n-3) + m7*r^(n-7) + m11*r^(n-11) ...
//
//          = r^4 * (m1*r^(n-4) + m4*r^(n-8) + m8 *r^(n-16) ... + mn_3)   // factor out r^4..r; same applies to 16 but r^16..r factors
//          + r^3 * (m2*r^(n-4) + m5*r^(n-8) + m9 *r^(n-16) ... + mn_2)
//          + r^2 * (m3*r^(n-4) + m6*r^(n-8) + m10*r^(n-16) ... + mn_1)
//          + r^1 * (m4*r^(n-4) + m7*r^(n-8) + m11*r^(n-16) ... + mn_0)   // Note last column: message group has no multiplier
//
//          = (((m1*r^4 + m4)*r^4 + m8 )*r^4 ... + mn_3) * r^4   // reverse Horner's rule, for each group
//          + (((m2*r^4 + m5)*r^4 + m9 )*r^4 ... + mn_2) * r^3   // each column is multiplied by r^4, except last
//          + (((m3*r^4 + m6)*r^4 + m10)*r^4 ... + mn_1) * r^2
//          + (((m4*r^4 + m7)*r^4 + m11)*r^4 ... + mn_0) * r^1
//
// Also see M. Goll and S. Gueron, "Vectorization of Poly1305 Message Authentication Code"
//
// Pseudocode:
//  * used for poly1305_multiply_scalar
//  x used for poly1305_multiply8_avx512
//  lower-case variables are scalar numbers in 3x44-bit limbs (in gprs)
//  upper-case variables are 8&16-element vector numbers in 3x44-bit limbs (in zmm registers)
//
//    CL = a       // [0 0 0 0 0 0 0 a]
//    AL = poly1305_limbs_avx512(input)
//    AH = poly1305_limbs_avx512(input+8)
//    AL = AL + C
//    input+=16, length-=16
//
//    a = r
//    a = a*r
//  r^2 = a
//    a = a*r
//  r^3 = a
//    r = a*r
//  r^4 = a
//
//    T  = r^4 || r^3 || r^2 || r
//    B  = limbs(T)           // [r^4  0  r^3  0  r^2  0  r^1  0 ]
//    CL = B >> 1             // [ 0  r^4  0  r^3  0  r^2  0  r^1]
//    R  = r^4 || r^4 || ..   // [r^4 r^4 r^4 r^4 r^4 r^4 r^4 r^4]
//    B  = BxR                // [r^8  0  r^7  0  r^6  0  r^5  0 ]
//    B  = B | CL             // [r^8 r^4 r^7 r^3 r^6 r^2 r^5 r^1]
//    CL = B
//    R  = r^8 || r^8 || ..   // [r^8 r^8 r^8 r^8 r^8 r^8 r^8 r^8]
//    B  = B x R              // [r^16 r^12 r^15 r^11 r^14 r^10 r^13 r^9]
//    CH = B
//    R = r^16 || r^16 || ..  // [r^16 r^16 r^16 r^16 r^16 r^16 r^16 r^16]
//
// for (;length>=16; input+=16, length-=16)
//     BL = poly1305_limbs_avx512(input)
//     BH = poly1305_limbs_avx512(input+8)
//     AL = AL x R
//     AH = AH x R
//     AL = AL + BL
//     AH = AH + BH
//
//  AL = AL x CL
//  AH = AH x CH
//  A = AL + AH // 16->8 blocks
//  T = A >> 4  // 8 ->4 blocks
//  A = A + T
//  T = A >> 2  // 4 ->2 blocks
//  A = A + T
//  T = A >> 1  // 2 ->1 blocks
//  A = A + T
//  a = A
//
// Register Map:
// GPRs:
//   input        = rdi
//   length       = rbx
//   accumulator  = rcx
//   R   = r8
//   a0  = rsi
//   a1  = r9
//   a2  = r10
//   r0  = r11
//   r1  = r12
//   c1  = r8;
//   t0  = r13
//   t1  = r14
//   t2  = r15
//   stack(rsp, rbp)
//   mulq(rax, rdx) in poly1305_multiply_scalar
//
// ZMMs:
//   D: xmm0-1
//   TMP: xmm2
//   T: xmm3-8
//   A: xmm9-14
//   B: xmm15-20
//   C: xmm21-26
//   R: xmm27-31
void StubGenerator::poly1305_process_blocks_avx512(
    const Register input, const Register length,
    const Register a0, const Register a1, const Register a2,
    const Register r0, const Register r1, const Register c1)
{
  Label L_process256Loop, L_process256LoopDone;
  const Register t0 = r13;
  const Register t1 = r14;
  const Register t2 = r15;
  const Register mulql = rax;
  const Register mulqh = rdx;

  const XMMRegister D0 = xmm0;
  const XMMRegister D1 = xmm1;
  const XMMRegister TMP = xmm2;

  const XMMRegister T0 = xmm3;
  const XMMRegister T1 = xmm4;
  const XMMRegister T2 = xmm5;
  const XMMRegister T3 = xmm6;
  const XMMRegister T4 = xmm7;
  const XMMRegister T5 = xmm8;

  const XMMRegister A0 = xmm9;
  const XMMRegister A1 = xmm10;
  const XMMRegister A2 = xmm11;
  const XMMRegister A3 = xmm12;
  const XMMRegister A4 = xmm13;
  const XMMRegister A5 = xmm14;

  const XMMRegister B0 = xmm15;
  const XMMRegister B1 = xmm16;
  const XMMRegister B2 = xmm17;
  const XMMRegister B3 = xmm18;
  const XMMRegister B4 = xmm19;
  const XMMRegister B5 = xmm20;

  const XMMRegister C0 = xmm21;
  const XMMRegister C1 = xmm22;
  const XMMRegister C2 = xmm23;
  const XMMRegister C3 = xmm24;
  const XMMRegister C4 = xmm25;
  const XMMRegister C5 = xmm26;

  const XMMRegister R0 = xmm27;
  const XMMRegister R1 = xmm28;
  const XMMRegister R2 = xmm29;
  const XMMRegister R1P = xmm30;
  const XMMRegister R2P = xmm31;

  // Spread accumulator into 44-bit limbs in quadwords C0,C1,C2
  __ movq(t0, a0);
  __ andq(t0, ExternalAddress(poly1305_mask44()), t1 /*rscratch*/); // First limb (Acc[43:0])
  __ movq(C0, t0);

  __ movq(t0, a1);
  __ shrdq(a0, t0, 44);
  __ andq(a0, ExternalAddress(poly1305_mask44()), t1 /*rscratch*/); // Second limb (Acc[77:52])
  __ movq(C1, a0);

  __ shrdq(a1, a2, 24);
  __ andq(a1, ExternalAddress(poly1305_mask42()), t1 /*rscratch*/); // Third limb (Acc[129:88])
  __ movq(C2, a1);

  // To add accumulator, we must unroll first loop iteration

  // Load first block of data (128 bytes) and pad
  // A0 to have bits 0-43 of all 8 blocks in 8 qwords
  // A1 to have bits 87-44 of all 8 blocks in 8 qwords
  // A2 to have bits 127-88 of all 8 blocks in 8 qwords
  __ evmovdquq(D0, Address(input, 0), Assembler::AVX_512bit);
  __ evmovdquq(D1, Address(input, 64), Assembler::AVX_512bit);
  poly1305_limbs_avx512(D0, D1, A0, A1, A2, true, TMP, t1 /*rscratch*/);

  // Add accumulator to the fist message block
  __ vpaddq(A0, A0, C0, Assembler::AVX_512bit);
  __ vpaddq(A1, A1, C1, Assembler::AVX_512bit);
  __ vpaddq(A2, A2, C2, Assembler::AVX_512bit);

  // Load next blocks of data (128 bytes)  and pad
  // A3 to have bits 0-43 of all 8 blocks in 8 qwords
  // A4 to have bits 87-44 of all 8 blocks in 8 qwords
  // A5 to have bits 127-88 of all 8 blocks in 8 qwords
  __ evmovdquq(D0, Address(input, 64*2), Assembler::AVX_512bit);
  __ evmovdquq(D1, Address(input, 64*3), Assembler::AVX_512bit);
  poly1305_limbs_avx512(D0, D1, A3, A4, A5, true, TMP, t1 /*rscratch*/);

  __ subl(length, 16*16);
  __ lea(input, Address(input,16*16));

  // Compute the powers of R^1..R^4 and form 44-bit limbs of each
  // T0 to have bits 0-127 in 4 quadword pairs
  // T1 to have bits 128-129 in alternating 8 qwords
  __ vpxorq(T1, T1, T1, Assembler::AVX_512bit);
  __ movq(T2, r0);
  __ vpinsrq(T2, T2, r1, 1);
  __ vinserti32x4(T0, T0, T2, 3);

  // Calculate R^2
  __ movq(a0, r0);
  __ movq(a1, r1);
  // "Clever": a2 not set because poly1305_multiply_scalar has a flag to indicate 128-bit accumulator
  poly1305_multiply_scalar(a0, a1, a2,
                           r0, r1, c1, true,
                           t0, t1, t2, mulql, mulqh);

  __ movq(T2, a0);
  __ vpinsrq(T2, T2, a1, 1);
  __ vinserti32x4(T0, T0, T2, 2);
  __ movq(T2, a2);
  __ vinserti32x4(T1, T1, T2, 2);

  // Calculate R^3
  poly1305_multiply_scalar(a0, a1, a2,
                           r0, r1, c1, false,
                           t0, t1, t2, mulql, mulqh);

  __ movq(T2, a0);
  __ vpinsrq(T2, T2, a1, 1);
  __ vinserti32x4(T0, T0, T2, 1);
  __ movq(T2, a2);
  __ vinserti32x4(T1, T1, T2, 1);

  // Calculate R^4
  poly1305_multiply_scalar(a0, a1, a2,
                           r0, r1, c1, false,
                           t0, t1, t2, mulql, mulqh);

  __ movq(T2, a0);
  __ vpinsrq(T2, T2, a1, 1);
  __ vinserti32x4(T0, T0, T2, 0);
  __ movq(T2, a2);
  __ vinserti32x4(T1, T1, T2, 0);

  // Interleave the powers of R^1..R^4 to form 44-bit limbs (half-empty)
  // B0 to have bits 0-43 of all 4 blocks in alternating 8 qwords
  // B1 to have bits 87-44 of all 4 blocks in alternating 8 qwords
  // B2 to have bits 127-88 of all 4 blocks in alternating 8 qwords
  __ vpxorq(T2, T2, T2, Assembler::AVX_512bit);
  poly1305_limbs_avx512(T0, T2, B0, B1, B2, false, TMP, t1 /*rscratch*/);

  // T1 contains the 2 highest bits of the powers of R
  __ vpsllq(T1, T1, 40, Assembler::AVX_512bit);
  __ evporq(B2, B2, T1, Assembler::AVX_512bit);

  // Broadcast 44-bit limbs of R^4 into R0,R1,R2
  __ mov(t0, a0);
  __ andq(t0, ExternalAddress(poly1305_mask44()), t1 /*rscratch*/); // First limb (R^4[43:0])
  __ evpbroadcastq(R0, t0, Assembler::AVX_512bit);

  __ movq(t0, a1);
  __ shrdq(a0, t0, 44);
  __ andq(a0, ExternalAddress(poly1305_mask44()), t1 /*rscratch*/); // Second limb (R^4[87:44])
  __ evpbroadcastq(R1, a0, Assembler::AVX_512bit);

  __ shrdq(a1, a2, 24);
  __ andq(a1, ExternalAddress(poly1305_mask42()), t1 /*rscratch*/); // Third limb (R^4[129:88])
  __ evpbroadcastq(R2, a1, Assembler::AVX_512bit);

  // Generate 4*5*R^4 into {R2P,R1P}
  // Used as multiplier in poly1305_multiply8_avx512 so can
  // ignore bottom limb and carry propagation
  __ vpsllq(R1P, R1, 2, Assembler::AVX_512bit);    // 4*R^4
  __ vpsllq(R2P, R2, 2, Assembler::AVX_512bit);
  __ vpaddq(R1P, R1P, R1, Assembler::AVX_512bit);  // 5*R^4
  __ vpaddq(R2P, R2P, R2, Assembler::AVX_512bit);
  __ vpsllq(R1P, R1P, 2, Assembler::AVX_512bit);   // 4*5*R^4
  __ vpsllq(R2P, R2P, 2, Assembler::AVX_512bit);

  // Move R^4..R^1 one element over
  __ vpslldq(C0, B0, 8, Assembler::AVX_512bit);
  __ vpslldq(C1, B1, 8, Assembler::AVX_512bit);
  __ vpslldq(C2, B2, 8, Assembler::AVX_512bit);

  // Calculate R^8-R^5
  poly1305_multiply8_avx512(B0, B1, B2,             // ACC=R^4..R^1
                            R0, R1, R2, R1P, R2P,   // R^4..R^4, 4*5*R^4
                            T0, T1, T2, T3, T4, T5, TMP, t1 /*rscratch*/);

  // Interleave powers of R: R^8 R^4 R^7 R^3 R^6 R^2 R^5 R
  __ evporq(B0, B0, C0, Assembler::AVX_512bit);
  __ evporq(B1, B1, C1, Assembler::AVX_512bit);
  __ evporq(B2, B2, C2, Assembler::AVX_512bit);

  // Store R^8-R for later use
  __ evmovdquq(C0, B0, Assembler::AVX_512bit);
  __ evmovdquq(C1, B1, Assembler::AVX_512bit);
  __ evmovdquq(C2, B2, Assembler::AVX_512bit);

  // Broadcast R^8
  __ vpbroadcastq(R0, B0, Assembler::AVX_512bit);
  __ vpbroadcastq(R1, B1, Assembler::AVX_512bit);
  __ vpbroadcastq(R2, B2, Assembler::AVX_512bit);

  // Generate 4*5*R^8
  __ vpsllq(R1P, R1, 2, Assembler::AVX_512bit);
  __ vpsllq(R2P, R2, 2, Assembler::AVX_512bit);
  __ vpaddq(R1P, R1P, R1, Assembler::AVX_512bit);    // 5*R^8
  __ vpaddq(R2P, R2P, R2, Assembler::AVX_512bit);
  __ vpsllq(R1P, R1P, 2, Assembler::AVX_512bit);     // 4*5*R^8
  __ vpsllq(R2P, R2P, 2, Assembler::AVX_512bit);

  // Calculate R^16-R^9
  poly1305_multiply8_avx512(B0, B1, B2,            // ACC=R^8..R^1
                            R0, R1, R2, R1P, R2P,  // R^8..R^8, 4*5*R^8
                            T0, T1, T2, T3, T4, T5, TMP, t1 /*rscratch*/);

  // Store R^16-R^9 for later use
  __ evmovdquq(C3, B0, Assembler::AVX_512bit);
  __ evmovdquq(C4, B1, Assembler::AVX_512bit);
  __ evmovdquq(C5, B2, Assembler::AVX_512bit);

  // Broadcast R^16
  __ vpbroadcastq(R0, B0, Assembler::AVX_512bit);
  __ vpbroadcastq(R1, B1, Assembler::AVX_512bit);
  __ vpbroadcastq(R2, B2, Assembler::AVX_512bit);

  // Generate 4*5*R^16
  __ vpsllq(R1P, R1, 2, Assembler::AVX_512bit);
  __ vpsllq(R2P, R2, 2, Assembler::AVX_512bit);
  __ vpaddq(R1P, R1P, R1, Assembler::AVX_512bit);  // 5*R^16
  __ vpaddq(R2P, R2P, R2, Assembler::AVX_512bit);
  __ vpsllq(R1P, R1P, 2, Assembler::AVX_512bit);   // 4*5*R^16
  __ vpsllq(R2P, R2P, 2, Assembler::AVX_512bit);

  // VECTOR LOOP: process 16 * 16-byte message block at a time
  __ bind(L_process256Loop);
  __ cmpl(length, 16*16);
  __ jcc(Assembler::less, L_process256LoopDone);

  // Load and interleave next block of data (128 bytes)
  __ evmovdquq(D0, Address(input, 0), Assembler::AVX_512bit);
  __ evmovdquq(D1, Address(input, 64), Assembler::AVX_512bit);
  poly1305_limbs_avx512(D0, D1, B0, B1, B2, true, TMP, t1 /*rscratch*/);

  // Load and interleave next block of data (128 bytes)
  __ evmovdquq(D0, Address(input, 64*2), Assembler::AVX_512bit);
  __ evmovdquq(D1, Address(input, 64*3), Assembler::AVX_512bit);
  poly1305_limbs_avx512(D0, D1, B3, B4, B5, true, TMP, t1 /*rscratch*/);

  poly1305_multiply8_avx512(A0, A1, A2,            // MSG/ACC 16 blocks
                            R0, R1, R2, R1P, R2P,  // R^16..R^16, 4*5*R^16
                            T0, T1, T2, T3, T4, T5, TMP, t1 /*rscratch*/);
  poly1305_multiply8_avx512(A3, A4, A5,            // MSG/ACC 16 blocks
                            R0, R1, R2, R1P, R2P,  // R^16..R^16, 4*5*R^16
                            T0, T1, T2, T3, T4, T5, TMP, t1 /*rscratch*/);

  __ vpaddq(A0, A0, B0, Assembler::AVX_512bit); // Add low 42-bit bits from new blocks to accumulator
  __ vpaddq(A1, A1, B1, Assembler::AVX_512bit); // Add medium 42-bit bits from new blocks to accumulator
  __ vpaddq(A2, A2, B2, Assembler::AVX_512bit); // Add highest bits from new blocks to accumulator
  __ vpaddq(A3, A3, B3, Assembler::AVX_512bit); // Add low 42-bit bits from new blocks to accumulator
  __ vpaddq(A4, A4, B4, Assembler::AVX_512bit); // Add medium 42-bit bits from new blocks to accumulator
  __ vpaddq(A5, A5, B5, Assembler::AVX_512bit); // Add highest bits from new blocks to accumulator

  __ subl(length, 16*16);
  __ lea(input, Address(input,16*16));
  __ jmp(L_process256Loop);

  __ bind(L_process256LoopDone);

  // Tail processing: Need to multiply ACC by R^16..R^1 and add it all up into a single scalar value
  // Generate 4*5*[R^16..R^9] (ignore lowest limb)
  // Use D0 ~ R1P, D1 ~ R2P for higher powers
  __ vpsllq(R1P, C4, 2, Assembler::AVX_512bit);
  __ vpsllq(R2P, C5, 2, Assembler::AVX_512bit);
  __ vpaddq(R1P, R1P, C4, Assembler::AVX_512bit);    // 5*R^8
  __ vpaddq(R2P, R2P, C5, Assembler::AVX_512bit);
  __ vpsllq(D0, R1P, 2, Assembler::AVX_512bit);      // 4*5*R^8
  __ vpsllq(D1, R2P, 2, Assembler::AVX_512bit);

  // Generate 4*5*[R^8..R^1] (ignore lowest limb)
  __ vpsllq(R1P, C1, 2, Assembler::AVX_512bit);
  __ vpsllq(R2P, C2, 2, Assembler::AVX_512bit);
  __ vpaddq(R1P, R1P, C1, Assembler::AVX_512bit);    // 5*R^8
  __ vpaddq(R2P, R2P, C2, Assembler::AVX_512bit);
  __ vpsllq(R1P, R1P, 2, Assembler::AVX_512bit);     // 4*5*R^8
  __ vpsllq(R2P, R2P, 2, Assembler::AVX_512bit);

  poly1305_multiply8_avx512(A0, A1, A2,            // MSG/ACC 16 blocks
                            C3, C4, C5, D0, D1,    // R^16-R^9, R1P, R2P
                            T0, T1, T2, T3, T4, T5, TMP, t1 /*rscratch*/);
  poly1305_multiply8_avx512(A3, A4, A5,            // MSG/ACC 16 blocks
                            C0, C1, C2, R1P, R2P,  // R^8-R, R1P, R2P
                            T0, T1, T2, T3, T4, T5, TMP, t1 /*rscratch*/);

  // Add all blocks (horizontally)
  // 16->8 blocks
  __ vpaddq(A0, A0, A3, Assembler::AVX_512bit);
  __ vpaddq(A1, A1, A4, Assembler::AVX_512bit);
  __ vpaddq(A2, A2, A5, Assembler::AVX_512bit);

  // 8 -> 4 blocks
  __ vextracti64x4(T0, A0, 1);
  __ vextracti64x4(T1, A1, 1);
  __ vextracti64x4(T2, A2, 1);
  __ vpaddq(A0, A0, T0, Assembler::AVX_256bit);
  __ vpaddq(A1, A1, T1, Assembler::AVX_256bit);
  __ vpaddq(A2, A2, T2, Assembler::AVX_256bit);

  // 4 -> 2 blocks
  __ vextracti32x4(T0, A0, 1);
  __ vextracti32x4(T1, A1, 1);
  __ vextracti32x4(T2, A2, 1);
  __ vpaddq(A0, A0, T0, Assembler::AVX_128bit);
  __ vpaddq(A1, A1, T1, Assembler::AVX_128bit);
  __ vpaddq(A2, A2, T2, Assembler::AVX_128bit);

  // 2 -> 1 blocks
  __ vpsrldq(T0, A0, 8, Assembler::AVX_128bit);
  __ vpsrldq(T1, A1, 8, Assembler::AVX_128bit);
  __ vpsrldq(T2, A2, 8, Assembler::AVX_128bit);

  // Finish folding and clear second qword
  __ mov64(t0, 0xfd);
  __ kmovql(k1, t0);
  __ evpaddq(A0, k1, A0, T0, false, Assembler::AVX_512bit);
  __ evpaddq(A1, k1, A1, T1, false, Assembler::AVX_512bit);
  __ evpaddq(A2, k1, A2, T2, false, Assembler::AVX_512bit);

  // Carry propagation
  __ vpsrlq(D0, A0, 44, Assembler::AVX_512bit);
  __ evpandq(A0, A0, ExternalAddress(poly1305_mask44()), Assembler::AVX_512bit, t1 /*rscratch*/); // Clear top 20 bits
  __ vpaddq(A1, A1, D0, Assembler::AVX_512bit);
  __ vpsrlq(D0, A1, 44, Assembler::AVX_512bit);
  __ evpandq(A1, A1, ExternalAddress(poly1305_mask44()), Assembler::AVX_512bit, t1 /*rscratch*/); // Clear top 20 bits
  __ vpaddq(A2, A2, D0, Assembler::AVX_512bit);
  __ vpsrlq(D0, A2, 42, Assembler::AVX_512bit);
  __ evpandq(A2, A2, ExternalAddress(poly1305_mask42()), Assembler::AVX_512bit, t1 /*rscratch*/); // Clear top 22 bits
  __ vpsllq(D1, D0, 2, Assembler::AVX_512bit);
  __ vpaddq(D0, D0, D1, Assembler::AVX_512bit);
  __ vpaddq(A0, A0, D0, Assembler::AVX_512bit);

  // Put together A (accumulator)
  __ movq(a0, A0);

  __ movq(t0, A1);
  __ movq(t1, t0);
  __ shlq(t1, 44);
  __ shrq(t0, 20);

  __ movq(a2, A2);
  __ movq(a1, a2);
  __ shlq(a1, 24);
  __ shrq(a2, 40);

  __ addq(a0, t1);
  __ adcq(a1, t0);
  __ adcq(a2, 0);

  // Cleanup
  // Zero out zmm0-zmm31.
  __ vzeroall();
  for (XMMRegister rxmm = xmm16; rxmm->is_valid(); rxmm = rxmm->successor()) {
    __ vpxorq(rxmm, rxmm, rxmm, Assembler::AVX_512bit);
  }
}

// This function consumes as many whole 16-byte blocks as available in input
// After execution, input and length will point at remaining (unprocessed) data
// and accumulator will point to the current accumulator value
address StubGenerator::generate_poly1305_processBlocks() {
  __ align(CodeEntryAlignment);
  StubGenStubId stub_id = StubGenStubId::poly1305_processBlocks_id;
  StubCodeMark mark(this, stub_id);
  address start = __ pc();
  __ enter();

  // Save all 'SOE' registers
  __ push(rbx);
  #ifdef _WIN64
  __ push(rsi);
  __ push(rdi);
  #endif
  __ push(r12);
  __ push(r13);
  __ push(r14);
  __ push(r15);

  // Register Map
  const Register input        = rdi; // msg
  const Register length       = rbx; // msg length in bytes
  const Register accumulator  = rcx;
  const Register R            = r8;

  const Register a0 = rsi;  // [in/out] accumulator bits 63..0
  const Register a1 = r9;   // [in/out] accumulator bits 127..64
  const Register a2 = r10;  // [in/out] accumulator bits 195..128
  const Register r0 = r11;  // R constant bits 63..0
  const Register r1 = r12;  // R constant bits 127..64
  const Register c1 = r8;   // 5*R (upper limb only)
  const Register t0 = r13;
  const Register t1 = r14;
  const Register t2 = r15;
  const Register mulql = rax;
  const Register mulqh = rdx;

  // Normalize input
  // pseudo-signature: void poly1305_processBlocks(byte[] input, int length, int[5] accumulator, int[5] R)
  // input, a, r pointers point at first array element
  // java headers bypassed in LibraryCallKit::inline_poly1305_processBlocks
  #ifdef _WIN64
  // c_rarg0 - rcx
  // c_rarg1 - rdx
  // c_rarg2 - r8
  // c_rarg3 - r9
  __ mov(input, c_rarg0);
  __ mov(length, c_rarg1);
  __ mov(accumulator, c_rarg2);
  __ mov(R, c_rarg3);
  #else
  // c_rarg0 - rdi
  // c_rarg1 - rsi
  // c_rarg2 - rdx
  // c_rarg3 - rcx
  // dont clobber R, args copied out-of-order
  __ mov(length, c_rarg1);
  __ mov(R, c_rarg3);
  __ mov(accumulator, c_rarg2);
  #endif

  Label L_process16Loop, L_process16LoopDone;

  // Load R into r1:r0
  poly1305_limbs(R, r0, r1, noreg, t0, t1);

  // Compute 5*R (Upper limb only)
  __ movq(c1, r1);
  __ shrq(c1, 2);
  __ addq(c1, r1); // c1 = r1 + (r1 >> 2)

  // Load accumulator into a2:a1:a0
  poly1305_limbs(accumulator, a0, a1, a2, t0, t1);

  // VECTOR LOOP: Minimum of 256 bytes to run vectorized code
  __ cmpl(length, 16*16);
  __ jcc(Assembler::less, L_process16Loop);

  if (UseAVX > 2) {
    poly1305_process_blocks_avx512(input, length,
                                  a0, a1, a2,
                                  r0, r1, c1);
  } else {
    poly1305_process_blocks_avx2(input, length,
                                  a0, a1, a2,
                                  r0, r1, c1);
  }


  // SCALAR LOOP: process one 16-byte message block at a time
  __ bind(L_process16Loop);
  __ cmpl(length, 16);
  __ jcc(Assembler::less, L_process16LoopDone);

  __ addq(a0, Address(input,0));
  __ adcq(a1, Address(input,8));
  __ adcq(a2,1);
  poly1305_multiply_scalar(a0, a1, a2,
                           r0, r1, c1, false,
                           t0, t1, t2, mulql, mulqh);

  __ subl(length, 16);
  __ lea(input, Address(input,16));
  __ jmp(L_process16Loop);
  __ bind(L_process16LoopDone);

  // Write output
  poly1305_limbs_out(a0, a1, a2, accumulator, t0, t1);

  __ pop(r15);
  __ pop(r14);
  __ pop(r13);
  __ pop(r12);
  #ifdef _WIN64
  __ pop(rdi);
  __ pop(rsi);
  #endif
  __ pop(rbx);

  __ leave();
  __ ret(0);
  return start;
}

/*
  The AVX2 implementation below is directly based on the AVX2 Poly1305 hash computation as
  implemented in Intel(R) Multi-Buffer Crypto for IPsec Library.
  (url: https://github.com/intel/intel-ipsec-mb/blob/main/lib/avx2_t3/poly_fma_avx2.asm)

  Additional references:
  [1] Goll M, Gueron S., "Vectorization of Poly1305 message authentication code",
      12th International Conference on Information Technology-New Generations,
      2015 Apr 13 (pp. 145-150). IEEE.
  [2] Bhattacharyya S, Sarkar P., "Improved SIMD implementation of Poly1305",
      IET Information Security. 2020 Sep;14(5):521-30.
  Note: a compact summary of the Goll-Gueron AVX2 algorithm developed in [1] is presented in [2].
  [3] Wikipedia, "Parallel evaluation of Horner's method",
      (url: https://en.wikipedia.org/wiki/Horner%27s_method)
 ----------------------------------------------------------

  Poly1305 AVX2 algorithm:
  Let the 32-byte one-time key be partitioned into two equal parts R and K.
  Let R be the 16-byte secret key used for polynomial evaluation.
  Let K be the 16-byte secret key.
  Let Z_P be prime field over which the polynomial is evaluated. Let P = 2^130 - 5 be the prime.
  Let M be the message which can be represented as a concatenation (||) of 'l' 16-byte blocks M[i].
  i.e., M = M[0] || M[1] || ... || M[i] || ... || M[l-2] || M[l-1]
  To create the coefficients C[i] for polynomial evaluation over Z_P, each 16-byte (i.e., 128-bit)
  message block M[i] is concatenated with bits '10' to make a 130-bit block.
  The last block (<= 16-byte length) is concatenated with 1 followed by 0s to make a 130-bit block.
  Therefore, we define
  C[i]   = M[i] || '10' for 0 <= i <= l-2 ;
  C[l-1] = M[i] || '10...0'
  such that, length(C[i]) = 130 bits, for i ∈ [0, l).

  Let * indicate scalar multiplication (i.e., w = u * v);
  Let × indicate scalar multiplication followed by reduction modulo P (i.e., z = u × v = {(u * v) mod P})

  POLY1305_MAC = (POLY1305_EVAL_POLYNOMIAL(C, R, P) + K) mod 2^128; where,

  POLY1305_EVAL_POLYNOMIAL(C, R, P) = {C[0] * R^l + C[1] * R^(l-1) + ... + C[l-2] * R^2 + C[l-1] * R} mod P
    = R × {C[0] × R^(l-1) + C[1] × R^(l-2) + ... + C[l-2] × R + C[l-1]}
    = R × Polynomial(R; C[0], C[1], ... ,C[l-2], C[l-1])
  Where,
  Polynomial(R; C[0], C[1], ... ,C[l-2], C[l-1]) = Σ{C[i] × R^(l-i-1)} for i ∈ [0, l)
  ----------------------------------------------------------

  Parallel evaluation of POLY1305_EVAL_POLYNOMIAL(C, R, P):
  Let the number of message blocks l = 4*l' + ρ where ρ = l mod 4.
  Using k-way parallel Horner's evaluation [3], for k = 4, we define SUM below:

  SUM = R^4 × Polynomial(R^4; C[0], C[4], C[8]  ... , C[4l'-4]) +
        R^3 × Polynomial(R^4; C[1], C[5], C[9]  ... , C[4l'-3]) +
        R^2 × Polynomial(R^4; C[2], C[6], C[10] ... , C[4l'-2]) +
        R^1 × Polynomial(R^4; C[3], C[7], C[11] ... , C[4l'-1]) +

  Then,
  POLY1305_EVAL_POLYNOMIAL(C, R, P) = SUM if ρ = 0 (i.e., l is multiple of 4)
                        = R × Polynomial(R; SUM + C[l-ρ], C[l-ρ+1], ... , C[l-1]) if ρ > 0
  ----------------------------------------------------------

  Gall-Gueron[1] 4-way SIMD Algorithm[2] for POLY1305_EVAL_POLYNOMIAL(C, R, P):

  Define mathematical vectors (not same as SIMD vector lanes) as below:
  R4321   = [R^4, R^3, R^2, R^1];
  R4444   = [R^4, R^4, R^4, R^4];
  COEF[i] = [C[4i], C[4i+1], C[4i+2], C[4i+3]] for i ∈ [0, l'). For example, COEF[0] and COEF[1] shown below.
  COEF[0] = [C0, C1, C2, C3]
  COEF[1] = [C4, C5, C6, C7]
  T       = [T0, T1, T2, T3] be a temporary vector
  ACC     = [acc, 0, 0, 0]; acc has hash from previous computations (if any), otherwise 0.
  ⊗ indicates component-wise vector multiplication followed by modulo reduction
  ⊕ indicates component-wise vector addition, + indicates scalar addition

  POLY1305_EVAL_POLYNOMIAL(C, R, P) {
    T ← ACC; # load accumulator
    T ← T ⊕ COEF[0]; # add accumulator to the first 4 blocks
    Compute R4321, R4444;
    # SIMD loop
    l' = floor(l/4); # operate on 4 blocks at a time
    for (i = 1 to l'-1):
      T ← (R4444 ⊗ T) ⊕ COEF[i];
    T ← R4321 ⊗ T;
    SUM ← T0 + T1 + T2 + T3;

    # Scalar tail processing
    if (ρ > 0):
      SUM ← R × Polynomial(R; SUM + C[l-ρ], C[l-ρ+1], ... , C[l-1]);
    return SUM;
  }

  Notes:
  (1) Each 130-bit block is represented using three 44-bit limbs (most significant limb is only 42-bit).
      (The Goll-Gueron implementation[1] uses five 26-bit limbs instead).
  (2) Each component of the mathematical vectors is a 130-bit value. The above mathemetical vectors are not to be confused with SIMD vector lanes.
  (3) Each AVX2 YMM register can store four 44-bit limbs in quadwords. Since each 130-bit message block is represented using 3 limbs,
      to store all the limbs of 4 different 130-bit message blocks, we need 3 YMM registers in total.
  (4) In the AVX2 implementation, multiplication followed by modulo reduction and addition are performed for 4 blocks at a time.

*/

void StubGenerator::poly1305_process_blocks_avx2(
    const Register input, const Register length,
    const Register a0, const Register a1, const Register a2,
    const Register r0, const Register r1, const Register c1)
{
  Label L_process256Loop, L_process256LoopDone;
  const Register t0 = r13;
  const Register t1 = r14;
  const Register t2 = r15;
  const Register mulql = rax;
  const Register mulqh = rdx;

  const XMMRegister YMM_ACC0 = xmm0;
  const XMMRegister YMM_ACC1 = xmm1;
  const XMMRegister YMM_ACC2 = xmm2;

  const XMMRegister YTMP1 = xmm3;
  const XMMRegister YTMP2 = xmm4;
  const XMMRegister YTMP3 = xmm5;
  const XMMRegister YTMP4 = xmm6;
  const XMMRegister YTMP5 = xmm7;
  const XMMRegister YTMP6 = xmm8;
  const XMMRegister YTMP7 = xmm9;
  const XMMRegister YTMP8 = xmm10;
  const XMMRegister YTMP9 = xmm11;
  const XMMRegister YTMP10 = xmm12;
  const XMMRegister YTMP11 = xmm13;
  const XMMRegister YTMP12 = xmm14;
  const XMMRegister YTMP13 = xmm15;

  const XMMRegister YMM_R0 = YTMP11;
  const XMMRegister YMM_R1 = YTMP12;
  const XMMRegister YMM_R2 = YTMP13;

  // XWORD aliases of YMM registers (for convenience)
  const XMMRegister XTMP1 = YTMP1;
  const XMMRegister XTMP2 = YTMP2;
  const XMMRegister XTMP3 = YTMP3;

  // Setup stack frame
  // Save rbp and rsp
  __ push(rbp);
  __ movq(rbp, rsp);
  // Align stack and reserve space
  __ andq(rsp, -32);
  __ subptr(rsp, 32*8);

  /* Compute the following steps of POLY1305_EVAL_POLYNOMIAL algorithm
    T ← ACC
    T ← T ⊕ COEF[0];
  */

  // Spread accumulator into 44-bit limbs in quadwords
  // Accumulator limbs to be stored in YTMP1,YTMP2,YTMP3
  // First limb (Acc[43:0])
  __ movq(t0, a0);
  __ andq(t0, ExternalAddress(poly1305_mask44()), t1 /*rscratch*/);
  __ movq(XTMP1, t0);
  // Second limb (Acc[87:44])
  __ movq(t0, a1);
  __ shrdq(a0, t0, 44);
  __ andq(a0, ExternalAddress(poly1305_mask44()), t1 /*rscratch*/);
  __ movq(XTMP2, a0);
  // Third limb (Acc[129:88])
  __ shrdq(a1, a2, 24);
  __ andq(a1, ExternalAddress(poly1305_mask42()), t1 /*rscratch*/);
  __ movq(XTMP3, a1);
  // --- end of spread accumulator

  // To add accumulator, we must unroll first loop iteration
  // Load first four 16-byte message blocks of data (64 bytes)
  __ vmovdqu(YTMP4, Address(input, 0));
  __ vmovdqu(YTMP5, Address(input, 32));

  // Interleave the input message data to form 44-bit limbs
  // YMM_ACC0 to have bits 0-43 of all 4 blocks in 4 qwords
  // YMM_ACC1 to have bits 87-44 of all 4 blocks in 4 qwords
  // YMM_ACC2 to have bits 127-88 of all 4 blocks in 4 qwords
  // Interleave blocks of data
  __ vpunpckhqdq(YMM_ACC2, YTMP4, YTMP5, Assembler::AVX_256bit);
  __ vpunpcklqdq(YMM_ACC0, YTMP4, YTMP5, Assembler::AVX_256bit);

  // Middle 44-bit limbs of new blocks
  __ vpsrlq(YMM_ACC1, YMM_ACC0, 44, Assembler::AVX_256bit);
  __ vpsllq(YTMP4, YMM_ACC2, 20, Assembler::AVX_256bit);
  __ vpor(YMM_ACC1, YMM_ACC1, YTMP4, Assembler::AVX_256bit);
  __ vpand(YMM_ACC1, YMM_ACC1, ExternalAddress(poly1305_mask44()), Assembler::AVX_256bit, t1);

  // Lowest 44-bit limbs of new blocks
  __ vpand(YMM_ACC0, YMM_ACC0, ExternalAddress(poly1305_mask44()), Assembler::AVX_256bit, t1);

  // Highest 42-bit limbs of new blocks; pad the msg with 2^128
  __ vpsrlq(YMM_ACC2, YMM_ACC2, 24, Assembler::AVX_256bit);

  // Add 2^128 to all 4 final qwords for the message
  __ vpor(YMM_ACC2, YMM_ACC2, ExternalAddress(poly1305_pad_msg()), Assembler::AVX_256bit, t1);
  // --- end of input interleaving and message padding

  // Add accumulator to the fist message block
  // Accumulator limbs in YTMP1,YTMP2,YTMP3
  __ vpaddq(YMM_ACC0, YMM_ACC0, YTMP1, Assembler::AVX_256bit);
  __ vpaddq(YMM_ACC1, YMM_ACC1, YTMP2, Assembler::AVX_256bit);
  __ vpaddq(YMM_ACC2, YMM_ACC2, YTMP3, Assembler::AVX_256bit);

  /* Compute the following steps of POLY1305_EVAL_POLYNOMIAL algorithm
    Compute R4321, R4444;
    R4321   = [R^4, R^3, R^2, R^1];
    R4444   = [R^4, R^4, R^4, R^4];
  */

  // Compute the powers of R^1..R^4 and form 44-bit limbs of each
  // YTMP5 to have bits 0-127 for R^1 and R^2
  // YTMP6 to have bits 128-129 for R^1 and R^2
  __ movq(XTMP1, r0);
  __ vpinsrq(XTMP1, XTMP1, r1, 1);
  __ vinserti128(YTMP5, YTMP5, XTMP1, 1);
  // clear registers
  __ vpxor(YTMP10, YTMP10, YTMP10, Assembler::AVX_256bit);
  __ vpxor(YTMP6, YTMP6, YTMP6, Assembler::AVX_256bit);

  // Calculate R^2
  // a ← R
  __ movq(a0, r0);
  __ movq(a1, r1);
  // a ← a * R = R^2
  poly1305_multiply_scalar(a0, a1, a2,
                           r0, r1, c1, true,
                           t0, t1, t2, mulql, mulqh);
  // Store R^2 in YTMP5, YTM6
  __ movq(XTMP1, a0);
  __ vpinsrq(XTMP1, XTMP1, a1, 1);
  __ vinserti128(YTMP5, YTMP5, XTMP1, 0);
  __ movq(XTMP1, a2);
  __ vinserti128(YTMP6, YTMP6, XTMP1, 0);

  // Calculate R^3
  // a ← a * R = R^3
  poly1305_multiply_scalar(a0, a1, a2,
                           r0, r1, c1, false,
                           t0, t1, t2, mulql, mulqh);
  // Store R^3 in YTMP7, YTM2
  __ movq(XTMP1, a0);
  __ vpinsrq(XTMP1, XTMP1, a1, 1);
  __ vinserti128(YTMP7, YTMP7, XTMP1, 1);
  __ movq(XTMP1, a2);
  __ vinserti128(YTMP2, YTMP2, XTMP1, 1);

  // Calculate R^4
  // a ← a * R = R^4
  poly1305_multiply_scalar(a0, a1, a2,
                           r0, r1, c1, false,
                           t0, t1, t2, mulql, mulqh);
  // Store R^4 in YTMP7, YTM2
  __ movq(XTMP1, a0);
  __ vpinsrq(XTMP1, XTMP1, a1, 1);
  __ vinserti128(YTMP7, YTMP7, XTMP1, 0);
  __ movq(XTMP1, a2);
  __ vinserti128(YTMP2, YTMP2, XTMP1, 0);

  // Interleave the powers of R^1..R^4 to form 44-bit limbs (half-empty)
  __ vpunpckhqdq(YMM_R2, YTMP5, YTMP10, Assembler::AVX_256bit);
  __ vpunpcklqdq(YMM_R0, YTMP5, YTMP10, Assembler::AVX_256bit);
  __ vpunpckhqdq(YTMP3, YTMP7, YTMP10, Assembler::AVX_256bit);
  __ vpunpcklqdq(YTMP4, YTMP7, YTMP10, Assembler::AVX_256bit);

  __ vpslldq(YMM_R2, YMM_R2, 8, Assembler::AVX_256bit);
  __ vpslldq(YTMP6, YTMP6, 8, Assembler::AVX_256bit);
  __ vpslldq(YMM_R0, YMM_R0, 8, Assembler::AVX_256bit);
  __ vpor(YMM_R2, YMM_R2, YTMP3, Assembler::AVX_256bit);
  __ vpor(YMM_R0, YMM_R0, YTMP4, Assembler::AVX_256bit);
  __ vpor(YTMP6, YTMP6, YTMP2, Assembler::AVX_256bit);
  // Move 2 MSbits to top 24 bits, to be OR'ed later
  __ vpsllq(YTMP6, YTMP6, 40, Assembler::AVX_256bit);

  __ vpsrlq(YMM_R1, YMM_R0, 44, Assembler::AVX_256bit);
  __ vpsllq(YTMP5, YMM_R2, 20, Assembler::AVX_256bit);
  __ vpor(YMM_R1, YMM_R1, YTMP5, Assembler::AVX_256bit);
  __ vpand(YMM_R1, YMM_R1, ExternalAddress(poly1305_mask44()), Assembler::AVX_256bit, t1);

  __ vpand(YMM_R0, YMM_R0, ExternalAddress(poly1305_mask44()), Assembler::AVX_256bit, t1);
  __ vpsrlq(YMM_R2, YMM_R2, 24, Assembler::AVX_256bit);

  __ vpor(YMM_R2, YMM_R2, YTMP6, Assembler::AVX_256bit);
  // YMM_R0, YMM_R1, YMM_R2 have the limbs of R^1, R^2, R^3, R^4

  // Store R^4-R on stack for later use
  int _r4_r1_save = 0;
  __ vmovdqu(Address(rsp, _r4_r1_save + 0), YMM_R0);
  __ vmovdqu(Address(rsp, _r4_r1_save + 32), YMM_R1);
  __ vmovdqu(Address(rsp, _r4_r1_save + 32*2), YMM_R2);

  // Broadcast 44-bit limbs of R^4
  __ mov(t0, a0);
  __ andq(t0, ExternalAddress(poly1305_mask44()), t1 /*rscratch*/); // First limb (R^4[43:0])
  __ movq(YMM_R0, t0);
  __ vpermq(YMM_R0, YMM_R0, 0x0, Assembler::AVX_256bit);

  __ movq(t0, a1);
  __ shrdq(a0, t0, 44);
  __ andq(a0, ExternalAddress(poly1305_mask44()), t1 /*rscratch*/); // Second limb (R^4[87:44])
  __ movq(YMM_R1, a0);
  __ vpermq(YMM_R1, YMM_R1, 0x0, Assembler::AVX_256bit);

  __ shrdq(a1, a2, 24);
  __ andq(a1, ExternalAddress(poly1305_mask42()), t1 /*rscratch*/); // Third limb (R^4[129:88])
  __ movq(YMM_R2, a1);
  __ vpermq(YMM_R2, YMM_R2, 0x0, Assembler::AVX_256bit);
  // YMM_R0, YMM_R1, YMM_R2 have the limbs of R^4, R^4, R^4, R^4

  // Generate 4*5*R^4
  // 4*R^4
  __ vpsllq(YTMP1, YMM_R1, 2, Assembler::AVX_256bit);
  __ vpsllq(YTMP2, YMM_R2, 2, Assembler::AVX_256bit);
  // 5*R^4
  __ vpaddq(YTMP1, YTMP1, YMM_R1, Assembler::AVX_256bit);
  __ vpaddq(YTMP2, YTMP2, YMM_R2, Assembler::AVX_256bit);
  // 4*5*R^4
  __ vpsllq(YTMP1, YTMP1, 2, Assembler::AVX_256bit);
  __ vpsllq(YTMP2, YTMP2, 2, Assembler::AVX_256bit);

  //Store broadcasted R^4 and 4*5*R^4 on stack for later use
  int _r4_save = 32*3;
  int _r4p_save = 32*6;
  __ vmovdqu(Address(rsp, _r4_save + 0), YMM_R0);
  __ vmovdqu(Address(rsp, _r4_save + 32), YMM_R1);
  __ vmovdqu(Address(rsp, _r4_save + 32*2), YMM_R2);
  __ vmovdqu(Address(rsp, _r4p_save), YTMP1);
  __ vmovdqu(Address(rsp, _r4p_save + 32), YTMP2);

  // Get the number of multiples of 4 message blocks (64 bytes) for vectorization
  __ movq(t0, length);
  __ andq(t0, 0xffffffc0); // 0xffffffffffffffc0 after sign extension

  // VECTOR LOOP: process 4 * 16-byte message blocks at a time
  __ bind(L_process256Loop);
  __ cmpl(t0, 16*4); //64 bytes (4 blocks at a time)
  __ jcc(Assembler::belowEqual, L_process256LoopDone);

  /*
    Compute the following steps of POLY1305_EVAL_POLYNOMIAL algorithm
    l' = floor(l/4)
    for (i = 1 to l'-1):
      T ← (R4444 ⊗ T) ⊕ COEF[i];
  */

  // Perform multiply and reduce while loading the next block and adding it in interleaved manner
  // The logic to advance the SIMD loop counter (i.e. length -= 64) is inside the function below.
  // The function below also includes the logic to load the next 4 blocks of data for efficient port utilization.
  poly1305_msg_mul_reduce_vec4_avx2(YMM_ACC0, YMM_ACC1, YMM_ACC2,
                          Address(rsp, _r4_save + 0), Address(rsp, _r4_save + 32), Address(rsp, _r4_save + 32*2),
                          Address(rsp, _r4p_save), Address(rsp, _r4p_save + 32),
                          YTMP1, YTMP2, YTMP3, YTMP4, YTMP5, YTMP6,
                          YTMP7, YTMP8, YTMP9, YTMP10, YTMP11, YTMP12,
                          input, t0, t1 /*rscratch*/);
  __ jmp(L_process256Loop);
  // end of vector loop
  __ bind(L_process256LoopDone);

  /*
    Compute the following steps of POLY1305_EVAL_POLYNOMIAL algorithm
    T ← R4321 ⊗ T;
  */

  // Need to multiply by R^4, R^3, R^2, R
  //Read R^4-R;
  __ vmovdqu(YMM_R0, Address(rsp, _r4_r1_save + 0));
  __ vmovdqu(YMM_R1, Address(rsp, _r4_r1_save + 32));
  __ vmovdqu(YMM_R2, Address(rsp, _r4_r1_save + 32*2));

  // Generate 4*5*[R^4..R^1] (ignore lowest limb)
  // YTMP1 to have bits 87-44 of all 1-4th powers of R' in 4 qwords
  // YTMP2 to have bits 129-88 of all 1-4th powers of R' in 4 qwords
  __ vpsllq(YTMP10, YMM_R1, 2, Assembler::AVX_256bit);
  __ vpaddq(YTMP1, YMM_R1, YTMP10, Assembler::AVX_256bit);  //R1' (R1*5)
  __ vpsllq(YTMP10, YMM_R2, 2, Assembler::AVX_256bit);
  __ vpaddq(YTMP2, YMM_R2, YTMP10, Assembler::AVX_256bit);  //R2' (R2*5)

  // 4*5*R
  __ vpsllq(YTMP1, YTMP1, 2, Assembler::AVX_256bit);
  __ vpsllq(YTMP2, YTMP2, 2, Assembler::AVX_256bit);

  poly1305_mul_reduce_vec4_avx2(YMM_ACC0, YMM_ACC1, YMM_ACC2,
                          YMM_R0, YMM_R1, YMM_R2, YTMP1, YTMP2,
                          YTMP3, YTMP4, YTMP5, YTMP6,
                          YTMP7, YTMP8, YTMP9, t1);
  /*
    Compute the following steps of POLY1305_EVAL_POLYNOMIAL algorithm
    SUM ← T0 + T1 + T2 + T3;
  */

  // 4 -> 2 blocks
  __ vextracti128(YTMP1, YMM_ACC0, 1);
  __ vextracti128(YTMP2, YMM_ACC1, 1);
  __ vextracti128(YTMP3, YMM_ACC2, 1);

  __ vpaddq(YMM_ACC0, YMM_ACC0, YTMP1, Assembler::AVX_128bit);
  __ vpaddq(YMM_ACC1, YMM_ACC1, YTMP2, Assembler::AVX_128bit);
  __ vpaddq(YMM_ACC2, YMM_ACC2, YTMP3, Assembler::AVX_128bit);
  // 2 -> 1 blocks
  __ vpsrldq(YTMP1, YMM_ACC0, 8, Assembler::AVX_128bit);
  __ vpsrldq(YTMP2, YMM_ACC1, 8, Assembler::AVX_128bit);
  __ vpsrldq(YTMP3, YMM_ACC2, 8, Assembler::AVX_128bit);

  // Finish folding
  __ vpaddq(YMM_ACC0, YMM_ACC0, YTMP1, Assembler::AVX_128bit);
  __ vpaddq(YMM_ACC1, YMM_ACC1, YTMP2, Assembler::AVX_128bit);
  __ vpaddq(YMM_ACC2, YMM_ACC2, YTMP3, Assembler::AVX_128bit);

  __ movq(YMM_ACC0, YMM_ACC0);
  __ movq(YMM_ACC1, YMM_ACC1);
  __ movq(YMM_ACC2, YMM_ACC2);

  __ lea(input, Address(input,16*4));
  __ andq(length, 63); // remaining bytes < length 64
  // carry propagation
  __ vpsrlq(YTMP1, YMM_ACC0, 44, Assembler::AVX_128bit);
  __ vpand(YMM_ACC0, YMM_ACC0, ExternalAddress(poly1305_mask44()), Assembler::AVX_128bit, t1); // Clear top 20 bits
  __ vpaddq(YMM_ACC1, YMM_ACC1, YTMP1, Assembler::AVX_128bit);
  __ vpsrlq(YTMP1, YMM_ACC1, 44, Assembler::AVX_128bit);
  __ vpand(YMM_ACC1, YMM_ACC1, ExternalAddress(poly1305_mask44()), Assembler::AVX_128bit, t1); // Clear top 20 bits
  __ vpaddq(YMM_ACC2, YMM_ACC2, YTMP1, Assembler::AVX_128bit);
  __ vpsrlq(YTMP1, YMM_ACC2, 42, Assembler::AVX_128bit);
  __ vpand(YMM_ACC2, YMM_ACC2, ExternalAddress(poly1305_mask42()), Assembler::AVX_128bit, t1); // Clear top 20 bits
  __ vpsllq(YTMP2, YTMP1, 2, Assembler::AVX_128bit);
  __ vpaddq(YTMP1, YTMP1, YTMP2, Assembler::AVX_128bit);
  __ vpaddq(YMM_ACC0, YMM_ACC0, YTMP1, Assembler::AVX_128bit);

  // Put together A
  __ movq(a0, YMM_ACC0);
  __ movq(t0, YMM_ACC1);
  __ movq(t1, t0);
  __ shlq(t1, 44);
  __ orq(a0, t1);
  __ shrq(t0, 20);
  __ movq(a2, YMM_ACC2);
  __ movq(a1, a2);
  __ shlq(a1, 24);
  __ orq(a1, t0);
  __ shrq(a2, 40);

  // cleanup
  __ vzeroall(); // clears all ymm registers (ymm0 through ymm15)

  // SAFE DATA (clear powers of R)
  __ vmovdqu(Address(rsp, _r4_r1_save + 0), YTMP1);
  __ vmovdqu(Address(rsp, _r4_r1_save + 32), YTMP1);
  __ vmovdqu(Address(rsp, _r4_r1_save + 32*2), YTMP1);
  __ vmovdqu(Address(rsp, _r4_save + 0), YTMP1);
  __ vmovdqu(Address(rsp, _r4_save + 32), YTMP1);
  __ vmovdqu(Address(rsp, _r4_save + 32*2), YTMP1);
  __ vmovdqu(Address(rsp, _r4p_save), YTMP1);
  __ vmovdqu(Address(rsp, _r4p_save + 32), YTMP1);

  // Save rbp and rsp; clear stack frame
    __ movq(rsp, rbp);
    __ pop(rbp);

}

// Compute component-wise product for 4 16-byte message  blocks,
// i.e. For each block, compute [a2 a1 a0] = [a2 a1 a0] x [r2 r1 r0]
//
// Each block/number is represented by 3 44-bit limb digits, start with multiplication
//
//      a2       a1       a0
// x    r2       r1       r0
// ----------------------------------
//     a2xr0    a1xr0    a0xr0
// +   a1xr1    a0xr1  5xa2xr1'     (r1' = r1<<2)
// +   a0xr2  5xa2xr2' 5xa1xr2'     (r2' = r2<<2)
// ----------------------------------
//        p2       p1       p0
//
void StubGenerator::poly1305_mul_reduce_vec4_avx2(
  const XMMRegister A0, const XMMRegister A1, const XMMRegister A2,
  const XMMRegister R0, const XMMRegister R1, const XMMRegister R2,
  const XMMRegister R1P, const XMMRegister R2P,
  const XMMRegister P0L, const XMMRegister P0H,
  const XMMRegister P1L, const XMMRegister P1H,
  const XMMRegister P2L, const XMMRegister P2H,
  const XMMRegister YTMP1, const Register rscratch)
{
  // Reset accumulator
  __ vpxor(P0L, P0L, P0L, Assembler::AVX_256bit);
  __ vpxor(P0H, P0H, P0H, Assembler::AVX_256bit);
  __ vpxor(P1L, P1L, P1L, Assembler::AVX_256bit);
  __ vpxor(P1H, P1H, P1H, Assembler::AVX_256bit);
  __ vpxor(P2L, P2L, P2L, Assembler::AVX_256bit);
  __ vpxor(P2H, P2H, P2H, Assembler::AVX_256bit);

  // Calculate partial products
  // p0 = a2xr1'
  // p1 = a2xr2'
  // p0 += a0xr0
  __ vpmadd52luq(P0L, A2, R1P, Assembler::AVX_256bit);
  __ vpmadd52huq(P0H, A2, R1P, Assembler::AVX_256bit);

  __ vpmadd52luq(P1L, A2, R2P, Assembler::AVX_256bit);
  __ vpmadd52huq(P1H, A2, R2P, Assembler::AVX_256bit);

  __ vpmadd52luq(P0L, A0, R0, Assembler::AVX_256bit);
  __ vpmadd52huq(P0H, A0, R0, Assembler::AVX_256bit);

  // p2 = a2xr0
  // p1 += a0xr1
  // p0 += a1xr2'
  // p2 += a0Xr2
  __ vpmadd52luq(P2L, A2, R0, Assembler::AVX_256bit);
  __ vpmadd52huq(P2H, A2, R0, Assembler::AVX_256bit);

  __ vpmadd52luq(P1L, A0, R1, Assembler::AVX_256bit);
  __ vpmadd52huq(P1H, A0, R1, Assembler::AVX_256bit);

  __ vpmadd52luq(P0L, A1, R2P, Assembler::AVX_256bit);
  __ vpmadd52huq(P0H, A1, R2P, Assembler::AVX_256bit);

  __ vpmadd52luq(P2L, A0, R2, Assembler::AVX_256bit);
  __ vpmadd52huq(P2H, A0, R2, Assembler::AVX_256bit);

  // Carry propgation (first pass)
  __ vpsrlq(YTMP1, P0L, 44, Assembler::AVX_256bit);
  __ vpsllq(P0H, P0H, 8, Assembler::AVX_256bit);
  __ vpmadd52luq(P1L, A1, R0, Assembler::AVX_256bit);
  __ vpmadd52huq(P1H, A1, R0, Assembler::AVX_256bit);
  // Carry propagation (first pass) - continue
  __ vpand(A0, P0L, ExternalAddress(poly1305_mask44()), Assembler::AVX_256bit, rscratch); // Clear top 20 bits
  __ vpaddq(P0H, P0H, YTMP1, Assembler::AVX_256bit);
  __ vpmadd52luq(P2L, A1, R1, Assembler::AVX_256bit);
  __ vpmadd52huq(P2H, A1, R1, Assembler::AVX_256bit);

  // Carry propagation (first pass) - continue 2
  __ vpaddq(P1L, P1L, P0H, Assembler::AVX_256bit);
  __ vpsllq(P1H, P1H, 8, Assembler::AVX_256bit);
  __ vpsrlq(YTMP1, P1L, 44, Assembler::AVX_256bit);
  __ vpand(A1, P1L, ExternalAddress(poly1305_mask44()), Assembler::AVX_256bit, rscratch); // Clear top 20 bits

  __ vpaddq(P2L, P2L, P1H, Assembler::AVX_256bit);
  __ vpaddq(P2L, P2L, YTMP1, Assembler::AVX_256bit);
  __ vpand(A2, P2L, ExternalAddress(poly1305_mask42()), Assembler::AVX_256bit, rscratch); // Clear top 22 bits
  __ vpsrlq(YTMP1, P2L, 42, Assembler::AVX_256bit);
  __ vpsllq(P2H, P2H, 10, Assembler::AVX_256bit);
  __ vpaddq(P2H, P2H, YTMP1, Assembler::AVX_256bit);

  // Carry propagation (second pass)
  // Multiply by 5 the highest bits (above 130 bits)
  __ vpaddq(A0, A0, P2H, Assembler::AVX_256bit);
  __ vpsllq(P2H, P2H, 2, Assembler::AVX_256bit);
  __ vpaddq(A0, A0, P2H, Assembler::AVX_256bit);

  __ vpsrlq(YTMP1, A0, 44, Assembler::AVX_256bit);
  __ vpand(A0, A0, ExternalAddress(poly1305_mask44()), Assembler::AVX_256bit, rscratch); // Clear top 20 bits
  __ vpaddq(A1, A1, YTMP1, Assembler::AVX_256bit);
}

// Compute component-wise product for 4 16-byte message  blocks and adds the next 4 blocks
// i.e. For each block, compute [a2 a1 a0] = [a2 a1 a0] x [r2 r1 r0],
// followed by [a2 a1 a0] += [n2 n1 n0], where n contains the next 4 blocks of the message.
//
// Each block/number is represented by 3 44-bit limb digits, start with multiplication
//
//      a2       a1       a0
// x    r2       r1       r0
// ----------------------------------
//     a2xr0    a1xr0    a0xr0
// +   a1xr1    a0xr1  5xa2xr1'     (r1' = r1<<2)
// +   a0xr2  5xa2xr2' 5xa1xr2'     (r2' = r2<<2)
// ----------------------------------
//        p2       p1       p0
//
void StubGenerator::poly1305_msg_mul_reduce_vec4_avx2(
  const XMMRegister A0, const XMMRegister A1, const XMMRegister A2,
  const Address R0, const Address R1, const Address R2,
  const Address R1P, const Address R2P,
  const XMMRegister P0L, const XMMRegister P0H,
  const XMMRegister P1L, const XMMRegister P1H,
  const XMMRegister P2L, const XMMRegister P2H,
  const XMMRegister YTMP1, const XMMRegister YTMP2,
  const XMMRegister YTMP3, const XMMRegister YTMP4,
  const XMMRegister YTMP5, const XMMRegister YTMP6,
  const Register input, const Register length, const Register rscratch)
{
  // Reset accumulator
  __ vpxor(P0L, P0L, P0L, Assembler::AVX_256bit);
  __ vpxor(P0H, P0H, P0H, Assembler::AVX_256bit);
  __ vpxor(P1L, P1L, P1L, Assembler::AVX_256bit);
  __ vpxor(P1H, P1H, P1H, Assembler::AVX_256bit);
  __ vpxor(P2L, P2L, P2L, Assembler::AVX_256bit);
  __ vpxor(P2H, P2H, P2H, Assembler::AVX_256bit);

  // Calculate partial products
  // p0 = a2xr1'
  // p1 = a2xr2'
  // p2 = a2xr0
      __ vpmadd52luq(P0L, A2, R1P, Assembler::AVX_256bit);
      __ vpmadd52huq(P0H, A2, R1P, Assembler::AVX_256bit);
  // Interleave input loading with hash computation
  __ lea(input, Address(input,16*4));
  __ subl(length, 16*4);
      __ vpmadd52luq(P1L, A2, R2P, Assembler::AVX_256bit);
      __ vpmadd52huq(P1H, A2, R2P, Assembler::AVX_256bit);
  // Load next block of data (64 bytes)
  __ vmovdqu(YTMP1, Address(input, 0));
  __ vmovdqu(YTMP2, Address(input, 32));
  // interleave new blocks of data
  __ vpunpckhqdq(YTMP3, YTMP1, YTMP2, Assembler::AVX_256bit);
  __ vpunpcklqdq(YTMP1, YTMP1, YTMP2, Assembler::AVX_256bit);
      __ vpmadd52luq(P0L, A0, R0, Assembler::AVX_256bit);
      __ vpmadd52huq(P0H, A0, R0, Assembler::AVX_256bit);
  // Highest 42-bit limbs of new blocks
  __ vpsrlq(YTMP6, YTMP3, 24, Assembler::AVX_256bit);
  __ vpor(YTMP6, YTMP6, ExternalAddress(poly1305_pad_msg()), Assembler::AVX_256bit, rscratch);

  //Middle 44-bit limbs of new blocks
  __ vpsrlq(YTMP2, YTMP1, 44, Assembler::AVX_256bit);
  __ vpsllq(YTMP4, YTMP3, 20, Assembler::AVX_256bit);
      // p2 = a2xr0
      __ vpmadd52luq(P2L, A2, R0, Assembler::AVX_256bit);
      __ vpmadd52huq(P2H, A2, R0, Assembler::AVX_256bit);
  __ vpor(YTMP2, YTMP2, YTMP4, Assembler::AVX_256bit);
  __ vpand(YTMP2, YTMP2, ExternalAddress(poly1305_mask44()), Assembler::AVX_256bit, rscratch);
  // Lowest 44-bit limbs of new blocks
  __ vpand(YTMP1, YTMP1, ExternalAddress(poly1305_mask44()), Assembler::AVX_256bit, rscratch);

      __ vpmadd52luq(P1L, A0, R1, Assembler::AVX_256bit);
      __ vpmadd52huq(P1H, A0, R1, Assembler::AVX_256bit);
      __ vpmadd52luq(P0L, A1, R2P, Assembler::AVX_256bit);
      __ vpmadd52huq(P0H, A1, R2P, Assembler::AVX_256bit);
      __ vpmadd52luq(P2L, A0, R2, Assembler::AVX_256bit);
      __ vpmadd52huq(P2H, A0, R2, Assembler::AVX_256bit);

  // Carry propgation (first pass)
  __ vpsrlq(YTMP5, P0L, 44, Assembler::AVX_256bit);
  __ vpsllq(P0H, P0H, 8, Assembler::AVX_256bit);
      __ vpmadd52luq(P1L, A1, R0, Assembler::AVX_256bit);
      __ vpmadd52huq(P1H, A1, R0, Assembler::AVX_256bit);
  // Carry propagation (first pass) - continue
  __ vpand(A0, P0L, ExternalAddress(poly1305_mask44()), Assembler::AVX_256bit, rscratch); // Clear top 20 bits
  __ vpaddq(P0H, P0H, YTMP5, Assembler::AVX_256bit);
      __ vpmadd52luq(P2L, A1, R1, Assembler::AVX_256bit);
      __ vpmadd52huq(P2H, A1, R1, Assembler::AVX_256bit);

  // Carry propagation (first pass) - continue 2
  __ vpaddq(P1L, P1L, P0H, Assembler::AVX_256bit);
  __ vpsllq(P1H, P1H, 8, Assembler::AVX_256bit);
  __ vpsrlq(YTMP5, P1L, 44, Assembler::AVX_256bit);
  __ vpand(A1, P1L, ExternalAddress(poly1305_mask44()), Assembler::AVX_256bit, rscratch); // Clear top 20 bits

  __ vpaddq(P2L, P2L, P1H, Assembler::AVX_256bit);
  __ vpaddq(P2L, P2L, YTMP5, Assembler::AVX_256bit);
  __ vpand(A2, P2L, ExternalAddress(poly1305_mask42()), Assembler::AVX_256bit, rscratch); // Clear top 22 bits
  __ vpaddq(A2, A2, YTMP6, Assembler::AVX_256bit); // Add highest bits from new blocks to accumulator
  __ vpsrlq(YTMP5, P2L, 42, Assembler::AVX_256bit);
  __ vpsllq(P2H, P2H, 10, Assembler::AVX_256bit);
  __ vpaddq(P2H, P2H, YTMP5, Assembler::AVX_256bit);

  // Carry propagation (second pass)
  // Multiply by 5 the highest bits (above 130 bits)
  __ vpaddq(A0, A0, P2H, Assembler::AVX_256bit);
  __ vpsllq(P2H, P2H, 2, Assembler::AVX_256bit);
  __ vpaddq(A0, A0, P2H, Assembler::AVX_256bit);

  __ vpsrlq(YTMP5, A0, 44, Assembler::AVX_256bit);
  __ vpand(A0, A0, ExternalAddress(poly1305_mask44()), Assembler::AVX_256bit, rscratch); // Clear top 20 bits
  __ vpaddq(A0, A0, YTMP1, Assembler::AVX_256bit); //Add low 42-bit bits from new blocks to accumulator
  __ vpaddq(A1, A1, YTMP2, Assembler::AVX_256bit); //Add medium 42-bit bits from new blocks to accumulator
  __ vpaddq(A1, A1, YTMP5, Assembler::AVX_256bit);
}
