averello/linux-stable-mirror
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2026-05-26 11:40:24 +02:00
Code Issues Packages Projects Releases Wiki Activity

lib/crypto: riscv/ghash: Migrate optimized code into library

Browse Source 
Remove the "ghash-riscv64-zvkg" crypto_shash algorithm.  Move the
corresponding assembly code into lib/crypto/, modify it to take the
length in blocks instead of bytes, and wire it up to the GHASH library.

This makes the GHASH library be optimized with the RISC-V Vector
Cryptography Extension.  It also greatly reduces the amount of
riscv-specific glue code that is needed, and it fixes the issue where
this optimized GHASH code was disabled by default.

Note that this RISC-V code has multiple opportunities for improvement,
such as adding more parallelism, providing an optimized multiplication
function, and directly supporting POLYVAL.  But for now, this commit
simply tweaks ghash_zvkg() slightly to make it compatible with the
library, then wires it up to ghash_blocks_arch().

ghash_preparekey_arch() is also implemented to store the copy of the raw
key needed by the vghsh.vv instruction.

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260319061723.1140720-13-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
This commit is contained in:
Eric Biggers
2026-03-18 23:17:13 -07:00
parent 73f315c15d
commit af413d71f0
8 changed files with 70 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
arch/riscv/crypto/Kconfig
-11
View File
@@ -17,17 +17,6 @@ config CRYPTO_AES_RISCV64
- Zvkb vector crypto extension (CTR)
- Zvkg vector crypto extension (XTS)
config CRYPTO_GHASH_RISCV64
tristate "Hash functions: GHASH"
depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
select CRYPTO_GCM
help
GCM GHASH function (NIST SP 800-38D)
Architecture: riscv64 using:
- Zvkg vector crypto extension
config CRYPTO_SM3_RISCV64
tristate "Hash functions: SM3 (ShangMi 3)"
depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
arch/riscv/crypto/Makefile
-3
View File
@@ -4,9 +4,6 @@ obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o
aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \
aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o
obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o
ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o
obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o
sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o
arch/riscv/crypto/ghash-riscv64-glue.c
-146
View File
@@ -1,146 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* GHASH using the RISC-V vector crypto extensions
*
* Copyright (C) 2023 VRULL GmbH
* Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
*
* Copyright (C) 2023 SiFive, Inc.
* Author: Jerry Shih <jerry.shih@sifive.com>
*/
#include <asm/simd.h>
#include <asm/vector.h>
#include <crypto/b128ops.h>
#include <crypto/gf128mul.h>
#include <crypto/ghash.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/simd.h>
#include <crypto/utils.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data,
size_t len);
struct riscv64_ghash_tfm_ctx {
be128 key;
};
struct riscv64_ghash_desc_ctx {
be128 accumulator;
};
static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
unsigned int keylen)
{
struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm);
if (keylen != GHASH_BLOCK_SIZE)
return -EINVAL;
memcpy(&tctx->key, key, GHASH_BLOCK_SIZE);
return 0;
}
static int riscv64_ghash_init(struct shash_desc *desc)
{
struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
*dctx = (struct riscv64_ghash_desc_ctx){};
return 0;
}
static inline void
riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx,
struct riscv64_ghash_desc_ctx *dctx,
const u8 *src, size_t srclen)
{
/* The srclen is nonzero and a multiple of 16. */
if (crypto_simd_usable()) {
kernel_vector_begin();
ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen);
kernel_vector_end();
} else {
do {
crypto_xor((u8 *)&dctx->accumulator, src,
GHASH_BLOCK_SIZE);
gf128mul_lle(&dctx->accumulator, &tctx->key);
src += GHASH_BLOCK_SIZE;
srclen -= GHASH_BLOCK_SIZE;
} while (srclen);
}
}
static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src,
unsigned int srclen)
{
const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
riscv64_ghash_blocks(tctx, dctx, src,
round_down(srclen, GHASH_BLOCK_SIZE));
return srclen - round_down(srclen, GHASH_BLOCK_SIZE);
}
static int riscv64_ghash_finup(struct shash_desc *desc, const u8 *src,
unsigned int len, u8 *out)
{
const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
if (len) {
u8 buf[GHASH_BLOCK_SIZE] = {};
memcpy(buf, src, len);
riscv64_ghash_blocks(tctx, dctx, buf, GHASH_BLOCK_SIZE);
memzero_explicit(buf, sizeof(buf));
}
memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE);
return 0;
}
static struct shash_alg riscv64_ghash_alg = {
.init = riscv64_ghash_init,
.update = riscv64_ghash_update,
.finup = riscv64_ghash_finup,
.setkey = riscv64_ghash_setkey,
.descsize = sizeof(struct riscv64_ghash_desc_ctx),
.digestsize = GHASH_DIGEST_SIZE,
.base = {
.cra_blocksize = GHASH_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx),
.cra_priority = 300,
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
.cra_name = "ghash",
.cra_driver_name = "ghash-riscv64-zvkg",
.cra_module = THIS_MODULE,
},
};
static int __init riscv64_ghash_mod_init(void)
{
if (riscv_isa_extension_available(NULL, ZVKG) &&
riscv_vector_vlen() >= 128)
return crypto_register_shash(&riscv64_ghash_alg);
return -ENODEV;
}
static void __exit riscv64_ghash_mod_exit(void)
{
crypto_unregister_shash(&riscv64_ghash_alg);
}
module_init(riscv64_ghash_mod_init);
module_exit(riscv64_ghash_mod_exit);
MODULE_DESCRIPTION("GHASH (RISC-V accelerated)");
MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("ghash");
include/crypto/gf128hash.h
+3
View File
@@ -44,6 +44,9 @@ struct ghash_key {
#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64)
/** @htable: GHASH key format used by the POWER8 assembly code */
u64 htable[4][2];
#elif defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_RISCV)
/** @h_raw: The hash key H, in GHASH format */
u8 h_raw[GHASH_BLOCK_SIZE];
#endif
/** @h: The hash key H, in POLYVAL format */
struct polyval_elem h;
lib/crypto/Kconfig
+2
View File
@@ -122,6 +122,8 @@ config CRYPTO_LIB_GF128HASH_ARCH
default y if ARM && KERNEL_MODE_NEON
default y if ARM64
default y if PPC64 && VSX
default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
default y if X86_64
config CRYPTO_LIB_MD5
lib/crypto/Makefile
+1
View File
@@ -173,6 +173,7 @@ targets += powerpc/ghashp8-ppc.S
OBJECT_FILES_NON_STANDARD_powerpc/ghashp8-ppc.o := y
endif
libgf128hash-$(CONFIG_RISCV) += riscv/ghash-riscv64-zvkg.o
libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o
endif # CONFIG_CRYPTO_LIB_GF128HASH_ARCH
lib/crypto/riscv/gf128hash.h
+57
View File
@@ -0,0 +1,57 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* GHASH, RISC-V optimized
*
* Copyright (C) 2023 VRULL GmbH
* Copyright (C) 2023 SiFive, Inc.
* Copyright 2026 Google LLC
*/
#include <asm/simd.h>
#include <asm/vector.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkg);
asmlinkage void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE],
const u8 key[GHASH_BLOCK_SIZE],
const u8 *data, size_t nblocks);
#define ghash_preparekey_arch ghash_preparekey_arch
static void ghash_preparekey_arch(struct ghash_key *key,
const u8 raw_key[GHASH_BLOCK_SIZE])
{
/* Save key in POLYVAL format for fallback */
ghash_key_to_polyval(raw_key, &key->h);
/* Save key in GHASH format for zvkg */
memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE);
}
#define ghash_blocks_arch ghash_blocks_arch
static void ghash_blocks_arch(struct polyval_elem *acc,
const struct ghash_key *key,
const u8 *data, size_t nblocks)
{
if (static_branch_likely(&have_zvkg) && likely(may_use_simd())) {
u8 ghash_acc[GHASH_BLOCK_SIZE];
polyval_acc_to_ghash(acc, ghash_acc);
kernel_vector_begin();
ghash_zvkg(ghash_acc, key->h_raw, data, nblocks);
kernel_vector_end();
ghash_acc_to_polyval(ghash_acc, acc);
memzero_explicit(ghash_acc, sizeof(ghash_acc));
} else {
ghash_blocks_generic(acc, &key->h, data, nblocks);
}
}
#define gf128hash_mod_init_arch gf128hash_mod_init_arch
static void gf128hash_mod_init_arch(void)
{
if (riscv_isa_extension_available(NULL, ZVKG) &&
riscv_vector_vlen() >= 128)
static_branch_enable(&have_zvkg);
}
arch/riscv/crypto/ghash-riscv64-zvkg.S → lib/crypto/riscv/ghash-riscv64-zvkg.S
+7 -6
View File
@@ -50,12 +50,13 @@
#define ACCUMULATOR a0
#define KEY a1
#define DATA a2
#define LEN a3
#define NBLOCKS a3
// void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data,
// size_t len);
// void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE],
// const u8 key[GHASH_BLOCK_SIZE],
// const u8 *data, size_t nblocks);
//
// |len| must be nonzero and a multiple of 16 (GHASH_BLOCK_SIZE).
// |nblocks| must be nonzero.
SYM_FUNC_START(ghash_zvkg)
vsetivli zero, 4, e32, m1, ta, ma
vle32.v v1, (ACCUMULATOR)
@@ -64,8 +65,8 @@ SYM_FUNC_START(ghash_zvkg)
vle32.v v3, (DATA)
vghsh.vv v1, v2, v3
addi DATA, DATA, 16
addi LEN, LEN, -16
bnez LEN, .Lnext_block
addi NBLOCKS, NBLOCKS, -1
bnez NBLOCKS, .Lnext_block
vse32.v v1, (ACCUMULATOR)
ret