Resguardo inicial

linux-7.2.3/drivers/crypto/nx/Kconfig

@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+config CRYPTO_DEV_NX_ENCRYPT
+	tristate "Encryption acceleration support on pSeries platform"
+	depends on PPC_PSERIES && IBMVIO && !CPU_LITTLE_ENDIAN
+	default y
+	select CRYPTO_AES
+	select CRYPTO_CCM
+	help
+	  Support for PowerPC Nest (NX) encryption acceleration. This
+	  module supports acceleration for AES and SHA2 algorithms on
+	  the pSeries platform.  If you choose 'M' here, this module
+	  will be called nx_crypto.
+
+config CRYPTO_DEV_NX_COMPRESS
+	tristate "Compression acceleration support"
+	default y
+	select CRYPTO_ALGAPI
+	select 842_DECOMPRESS
+	help
+	  Support for PowerPC Nest (NX) compression acceleration. This
+	  module supports acceleration for compressing memory with the 842
+	  algorithm using the cryptographic API.  One of the platform
+	  drivers must be selected also.  If you choose 'M' here, this
+	  module will be called nx_compress.
+
+if CRYPTO_DEV_NX_COMPRESS
+
+config CRYPTO_DEV_NX_COMPRESS_PSERIES
+	tristate "Compression acceleration support on pSeries platform"
+	depends on PPC_PSERIES && IBMVIO
+	depends on PPC_VAS
+	default y
+	help
+	  Support for PowerPC Nest (NX) compression acceleration. This
+	  module supports acceleration for compressing memory with the 842
+	  algorithm.  This supports NX hardware on the pSeries platform.
+	  If you choose 'M' here, this module will be called nx_compress_pseries.
+
+config CRYPTO_DEV_NX_COMPRESS_POWERNV
+	tristate "Compression acceleration support on PowerNV platform"
+	depends on PPC_POWERNV
+	depends on PPC_VAS
+	default y
+	help
+	  Support for PowerPC Nest (NX) compression acceleration. This
+	  module supports acceleration for compressing memory with the 842
+	  algorithm.  This supports NX hardware on the PowerNV platform.
+	  If you choose 'M' here, this module will be called nx_compress_powernv.
+
+endif

linux-7.2.3/drivers/crypto/nx/Makefile

@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_CRYPTO_DEV_NX_ENCRYPT) += nx-crypto.o
+nx-crypto-objs := nx.o \
+		  nx-aes-cbc.o \
+		  nx-aes-ecb.o \
+		  nx-aes-gcm.o \
+		  nx-aes-ccm.o \
+		  nx-aes-ctr.o \
+		  nx-aes-xcbc.o \
+		  nx-sha256.o \
+		  nx-sha512.o
+
+nx-crypto-$(CONFIG_DEBUG_FS) += nx_debugfs.o
+obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_PSERIES) += nx-compress-pseries.o nx-compress.o
+obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_POWERNV) += nx-compress-powernv.o nx-compress.o
+nx-compress-objs := nx-842.o
+nx-compress-pseries-objs := nx-common-pseries.o
+nx-compress-powernv-objs := nx-common-powernv.o

linux-7.2.3/drivers/crypto/nx/nx-842.c

@@ -0,0 +1,526 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Cryptographic API for the NX-842 hardware compression.
+ *
+ * Copyright (C) IBM Corporation, 2011-2015
+ *
+ * Designer of the Power data compression engine:
+ *   Bulent Abali <abali@us.ibm.com>
+ *
+ * Original Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
+ *                   Seth Jennings <sjenning@linux.vnet.ibm.com>
+ *
+ * Rewrite: Dan Streetman <ddstreet@ieee.org>
+ *
+ * This is an interface to the NX-842 compression hardware in PowerPC
+ * processors.  Most of the complexity of this drvier is due to the fact that
+ * the NX-842 compression hardware requires the input and output data buffers
+ * to be specifically aligned, to be a specific multiple in length, and within
+ * specific minimum and maximum lengths.  Those restrictions, provided by the
+ * nx-842 driver via nx842_constraints, mean this driver must use bounce
+ * buffers and headers to correct misaligned in or out buffers, and to split
+ * input buffers that are too large.
+ *
+ * This driver will fall back to software decompression if the hardware
+ * decompression fails, so this driver's decompression should never fail as
+ * long as the provided compressed buffer is valid.  Any compressed buffer
+ * created by this driver will have a header (except ones where the input
+ * perfectly matches the constraints); so users of this driver cannot simply
+ * pass a compressed buffer created by this driver over to the 842 software
+ * decompression library.  Instead, users must use this driver to decompress;
+ * if the hardware fails or is unavailable, the compressed buffer will be
+ * parsed and the header removed, and the raw 842 buffer(s) passed to the 842
+ * software decompression library.
+ *
+ * This does not fall back to software compression, however, since the caller
+ * of this function is specifically requesting hardware compression; if the
+ * hardware compression fails, the caller can fall back to software
+ * compression, and the raw 842 compressed buffer that the software compressor
+ * creates can be passed to this driver for hardware decompression; any
+ * buffer without our specific header magic is assumed to be a raw 842 buffer
+ * and passed directly to the hardware.  Note that the software compression
+ * library will produce a compressed buffer that is incompatible with the
+ * hardware decompressor if the original input buffer length is not a multiple
+ * of 8; if such a compressed buffer is passed to this driver for
+ * decompression, the hardware will reject it and this driver will then pass
+ * it over to the software library for decompression.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/vmalloc.h>
+#include <linux/sw842.h>
+#include <linux/spinlock.h>
+
+#include "nx-842.h"
+
+/* The first 5 bits of this magic are 0x1f, which is an invalid 842 5-bit
+ * template (see lib/842/842.h), so this magic number will never appear at
+ * the start of a raw 842 compressed buffer.  That is important, as any buffer
+ * passed to us without this magic is assumed to be a raw 842 compressed
+ * buffer, and passed directly to the hardware to decompress.
+ */
+#define NX842_CRYPTO_MAGIC	(0xf842)
+#define NX842_CRYPTO_HEADER_SIZE(g)				\
+	(sizeof(struct nx842_crypto_header) +			\
+	 sizeof(struct nx842_crypto_header_group) * (g))
+#define NX842_CRYPTO_HEADER_MAX_SIZE				\
+	NX842_CRYPTO_HEADER_SIZE(NX842_CRYPTO_GROUP_MAX)
+
+/* bounce buffer size */
+#define BOUNCE_BUFFER_ORDER	(2)
+#define BOUNCE_BUFFER_SIZE					\
+	((unsigned int)(PAGE_SIZE << BOUNCE_BUFFER_ORDER))
+
+/* try longer on comp because we can fallback to sw decomp if hw is busy */
+#define COMP_BUSY_TIMEOUT	(250) /* ms */
+#define DECOMP_BUSY_TIMEOUT	(50) /* ms */
+
+struct nx842_crypto_param {
+	u8 *in;
+	unsigned int iremain;
+	u8 *out;
+	unsigned int oremain;
+	unsigned int ototal;
+};
+
+static int update_param(struct nx842_crypto_param *p,
+			unsigned int slen, unsigned int dlen)
+{
+	if (p->iremain < slen)
+		return -EOVERFLOW;
+	if (p->oremain < dlen)
+		return -ENOSPC;
+
+	p->in += slen;
+	p->iremain -= slen;
+	p->out += dlen;
+	p->oremain -= dlen;
+	p->ototal += dlen;
+
+	return 0;
+}
+
+void *nx842_crypto_alloc_ctx(struct nx842_driver *driver)
+{
+	struct nx842_crypto_ctx *ctx;
+
+	ctx = kzalloc_obj(*ctx);
+	if (!ctx)
+		return ERR_PTR(-ENOMEM);
+
+	spin_lock_init(&ctx->lock);
+	ctx->driver = driver;
+	ctx->wmem = kmalloc(driver->workmem_size, GFP_KERNEL);
+	ctx->sbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);
+	ctx->dbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);
+	if (!ctx->wmem || !ctx->sbounce || !ctx->dbounce) {
+		nx842_crypto_free_ctx(ctx);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	return ctx;
+}
+EXPORT_SYMBOL_GPL(nx842_crypto_alloc_ctx);
+
+void nx842_crypto_free_ctx(void *p)
+{
+	struct nx842_crypto_ctx *ctx = p;
+
+	kfree(ctx->wmem);
+	free_pages((unsigned long)ctx->sbounce, BOUNCE_BUFFER_ORDER);
+	free_pages((unsigned long)ctx->dbounce, BOUNCE_BUFFER_ORDER);
+	kfree(ctx);
+}
+EXPORT_SYMBOL_GPL(nx842_crypto_free_ctx);
+
+static void check_constraints(struct nx842_constraints *c)
+{
+	/* limit maximum, to always have enough bounce buffer to decompress */
+	if (c->maximum > BOUNCE_BUFFER_SIZE)
+		c->maximum = BOUNCE_BUFFER_SIZE;
+}
+
+static int nx842_crypto_add_header(struct nx842_crypto_header *hdr, u8 *buf)
+{
+	int s = NX842_CRYPTO_HEADER_SIZE(hdr->groups);
+
+	/* compress should have added space for header */
+	if (s > be16_to_cpu(hdr->group[0].padding)) {
+		pr_err("Internal error: no space for header\n");
+		return -EINVAL;
+	}
+
+	memcpy(buf, hdr, s);
+
+	print_hex_dump_debug("header ", DUMP_PREFIX_OFFSET, 16, 1, buf, s, 0);
+
+	return 0;
+}
+
+static int compress(struct nx842_crypto_ctx *ctx,
+		    struct nx842_crypto_param *p,
+		    struct nx842_crypto_header_group *g,
+		    struct nx842_constraints *c,
+		    u16 *ignore,
+		    unsigned int hdrsize)
+{
+	unsigned int slen = p->iremain, dlen = p->oremain, tmplen;
+	unsigned int adj_slen = slen;
+	u8 *src = p->in, *dst = p->out;
+	int ret, dskip = 0;
+	ktime_t timeout;
+
+	if (p->iremain == 0)
+		return -EOVERFLOW;
+
+	if (p->oremain == 0 || hdrsize + c->minimum > dlen)
+		return -ENOSPC;
+
+	if (slen % c->multiple)
+		adj_slen = round_up(slen, c->multiple);
+	if (slen < c->minimum)
+		adj_slen = c->minimum;
+	if (slen > c->maximum)
+		adj_slen = slen = c->maximum;
+	if (adj_slen > slen || (u64)src % c->alignment) {
+		adj_slen = min(adj_slen, BOUNCE_BUFFER_SIZE);
+		slen = min(slen, BOUNCE_BUFFER_SIZE);
+		if (adj_slen > slen)
+			memset(ctx->sbounce + slen, 0, adj_slen - slen);
+		memcpy(ctx->sbounce, src, slen);
+		src = ctx->sbounce;
+		slen = adj_slen;
+		pr_debug("using comp sbounce buffer, len %x\n", slen);
+	}
+
+	dst += hdrsize;
+	dlen -= hdrsize;
+
+	if ((u64)dst % c->alignment) {
+		dskip = (int)(PTR_ALIGN(dst, c->alignment) - dst);
+		dst += dskip;
+		dlen -= dskip;
+	}
+	if (dlen % c->multiple)
+		dlen = round_down(dlen, c->multiple);
+	if (dlen < c->minimum) {
+nospc:
+		dst = ctx->dbounce;
+		dlen = min(p->oremain, BOUNCE_BUFFER_SIZE);
+		dlen = round_down(dlen, c->multiple);
+		dskip = 0;
+		pr_debug("using comp dbounce buffer, len %x\n", dlen);
+	}
+	if (dlen > c->maximum)
+		dlen = c->maximum;
+
+	tmplen = dlen;
+	timeout = ktime_add_ms(ktime_get(), COMP_BUSY_TIMEOUT);
+	do {
+		dlen = tmplen; /* reset dlen, if we're retrying */
+		ret = ctx->driver->compress(src, slen, dst, &dlen, ctx->wmem);
+		/* possibly we should reduce the slen here, instead of
+		 * retrying with the dbounce buffer?
+		 */
+		if (ret == -ENOSPC && dst != ctx->dbounce)
+			goto nospc;
+	} while (ret == -EBUSY && ktime_before(ktime_get(), timeout));
+	if (ret)
+		return ret;
+
+	dskip += hdrsize;
+
+	if (dst == ctx->dbounce)
+		memcpy(p->out + dskip, dst, dlen);
+
+	g->padding = cpu_to_be16(dskip);
+	g->compressed_length = cpu_to_be32(dlen);
+	g->uncompressed_length = cpu_to_be32(slen);
+
+	if (p->iremain < slen) {
+		*ignore = slen - p->iremain;
+		slen = p->iremain;
+	}
+
+	pr_debug("compress slen %x ignore %x dlen %x padding %x\n",
+		 slen, *ignore, dlen, dskip);
+
+	return update_param(p, slen, dskip + dlen);
+}
+
+int nx842_crypto_compress(struct crypto_scomp *tfm,
+			  const u8 *src, unsigned int slen,
+			  u8 *dst, unsigned int *dlen, void *pctx)
+{
+	struct nx842_crypto_ctx *ctx = pctx;
+	struct nx842_crypto_header *hdr =
+				container_of(&ctx->header,
+					     struct nx842_crypto_header, hdr);
+	struct nx842_crypto_param p;
+	struct nx842_constraints c = *ctx->driver->constraints;
+	unsigned int groups, hdrsize, h;
+	int ret, n;
+	bool add_header;
+	u16 ignore = 0;
+
+	check_constraints(&c);
+
+	p.in = (u8 *)src;
+	p.iremain = slen;
+	p.out = dst;
+	p.oremain = *dlen;
+	p.ototal = 0;
+
+	*dlen = 0;
+
+	groups = min_t(unsigned int, NX842_CRYPTO_GROUP_MAX,
+		       DIV_ROUND_UP(p.iremain, c.maximum));
+	hdrsize = NX842_CRYPTO_HEADER_SIZE(groups);
+
+	spin_lock_bh(&ctx->lock);
+
+	/* skip adding header if the buffers meet all constraints */
+	add_header = (p.iremain % c.multiple	||
+		      p.iremain < c.minimum	||
+		      p.iremain > c.maximum	||
+		      (u64)p.in % c.alignment	||
+		      p.oremain % c.multiple	||
+		      p.oremain < c.minimum	||
+		      p.oremain > c.maximum	||
+		      (u64)p.out % c.alignment);
+
+	hdr->magic = cpu_to_be16(NX842_CRYPTO_MAGIC);
+	hdr->groups = 0;
+	hdr->ignore = 0;
+
+	while (p.iremain > 0) {
+		n = hdr->groups++;
+		ret = -ENOSPC;
+		if (hdr->groups > NX842_CRYPTO_GROUP_MAX)
+			goto unlock;
+
+		/* header goes before first group */
+		h = !n && add_header ? hdrsize : 0;
+
+		if (ignore)
+			pr_warn("internal error, ignore is set %x\n", ignore);
+
+		ret = compress(ctx, &p, &hdr->group[n], &c, &ignore, h);
+		if (ret)
+			goto unlock;
+	}
+
+	if (!add_header && hdr->groups > 1) {
+		pr_err("Internal error: No header but multiple groups\n");
+		ret = -EINVAL;
+		goto unlock;
+	}
+
+	/* ignore indicates the input stream needed to be padded */
+	hdr->ignore = cpu_to_be16(ignore);
+	if (ignore)
+		pr_debug("marked %d bytes as ignore\n", ignore);
+
+	if (add_header)
+		ret = nx842_crypto_add_header(hdr, dst);
+	if (ret)
+		goto unlock;
+
+	*dlen = p.ototal;
+
+	pr_debug("compress total slen %x dlen %x\n", slen, *dlen);
+
+unlock:
+	spin_unlock_bh(&ctx->lock);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(nx842_crypto_compress);
+
+static int decompress(struct nx842_crypto_ctx *ctx,
+		      struct nx842_crypto_param *p,
+		      struct nx842_crypto_header_group *g,
+		      struct nx842_constraints *c,
+		      u16 ignore)
+{
+	unsigned int slen = be32_to_cpu(g->compressed_length);
+	unsigned int required_len = be32_to_cpu(g->uncompressed_length);
+	unsigned int dlen = p->oremain, tmplen;
+	unsigned int adj_slen = slen;
+	u8 *src = p->in, *dst = p->out;
+	u16 padding = be16_to_cpu(g->padding);
+	int ret, spadding = 0;
+	ktime_t timeout;
+
+	if (!slen || !required_len)
+		return -EINVAL;
+
+	if (p->iremain <= 0 || padding + slen > p->iremain)
+		return -EOVERFLOW;
+
+	if (p->oremain <= 0 || required_len - ignore > p->oremain)
+		return -ENOSPC;
+
+	src += padding;
+
+	if (slen % c->multiple)
+		adj_slen = round_up(slen, c->multiple);
+	if (slen < c->minimum)
+		adj_slen = c->minimum;
+	if (slen > c->maximum)
+		goto usesw;
+	if (slen < adj_slen || (u64)src % c->alignment) {
+		/* we can append padding bytes because the 842 format defines
+		 * an "end" template (see lib/842/842_decompress.c) and will
+		 * ignore any bytes following it.
+		 */
+		if (slen < adj_slen)
+			memset(ctx->sbounce + slen, 0, adj_slen - slen);
+		memcpy(ctx->sbounce, src, slen);
+		src = ctx->sbounce;
+		spadding = adj_slen - slen;
+		slen = adj_slen;
+		pr_debug("using decomp sbounce buffer, len %x\n", slen);
+	}
+
+	if (dlen % c->multiple)
+		dlen = round_down(dlen, c->multiple);
+	if (dlen < required_len || (u64)dst % c->alignment) {
+		dst = ctx->dbounce;
+		dlen = min(required_len, BOUNCE_BUFFER_SIZE);
+		pr_debug("using decomp dbounce buffer, len %x\n", dlen);
+	}
+	if (dlen < c->minimum)
+		goto usesw;
+	if (dlen > c->maximum)
+		dlen = c->maximum;
+
+	tmplen = dlen;
+	timeout = ktime_add_ms(ktime_get(), DECOMP_BUSY_TIMEOUT);
+	do {
+		dlen = tmplen; /* reset dlen, if we're retrying */
+		ret = ctx->driver->decompress(src, slen, dst, &dlen, ctx->wmem);
+	} while (ret == -EBUSY && ktime_before(ktime_get(), timeout));
+	if (ret) {
+usesw:
+		/* reset everything, sw doesn't have constraints */
+		src = p->in + padding;
+		slen = be32_to_cpu(g->compressed_length);
+		spadding = 0;
+		dst = p->out;
+		dlen = p->oremain;
+		if (dlen < required_len) { /* have ignore bytes */
+			dst = ctx->dbounce;
+			dlen = BOUNCE_BUFFER_SIZE;
+		}
+		pr_info_ratelimited("using software 842 decompression\n");
+		ret = sw842_decompress(src, slen, dst, &dlen);
+	}
+	if (ret)
+		return ret;
+
+	slen -= spadding;
+
+	dlen -= ignore;
+	if (ignore)
+		pr_debug("ignoring last %x bytes\n", ignore);
+
+	if (dst == ctx->dbounce)
+		memcpy(p->out, dst, dlen);
+
+	pr_debug("decompress slen %x padding %x dlen %x ignore %x\n",
+		 slen, padding, dlen, ignore);
+
+	return update_param(p, slen + padding, dlen);
+}
+
+int nx842_crypto_decompress(struct crypto_scomp *tfm,
+			    const u8 *src, unsigned int slen,
+			    u8 *dst, unsigned int *dlen, void *pctx)
+{
+	struct nx842_crypto_ctx *ctx = pctx;
+	struct nx842_crypto_header *hdr;
+	struct nx842_crypto_param p;
+	struct nx842_constraints c = *ctx->driver->constraints;
+	int n, ret, hdr_len;
+	u16 ignore = 0;
+
+	check_constraints(&c);
+
+	p.in = (u8 *)src;
+	p.iremain = slen;
+	p.out = dst;
+	p.oremain = *dlen;
+	p.ototal = 0;
+
+	*dlen = 0;
+
+	hdr = (struct nx842_crypto_header *)src;
+
+	spin_lock_bh(&ctx->lock);
+
+	/* If it doesn't start with our header magic number, assume it's a raw
+	 * 842 compressed buffer and pass it directly to the hardware driver
+	 */
+	if (be16_to_cpu(hdr->magic) != NX842_CRYPTO_MAGIC) {
+		struct nx842_crypto_header_group g = {
+			.padding =		0,
+			.compressed_length =	cpu_to_be32(p.iremain),
+			.uncompressed_length =	cpu_to_be32(p.oremain),
+		};
+
+		ret = decompress(ctx, &p, &g, &c, 0);
+		if (ret)
+			goto unlock;
+
+		goto success;
+	}
+
+	if (!hdr->groups) {
+		pr_err("header has no groups\n");
+		ret = -EINVAL;
+		goto unlock;
+	}
+	if (hdr->groups > NX842_CRYPTO_GROUP_MAX) {
+		pr_err("header has too many groups %x, max %x\n",
+		       hdr->groups, NX842_CRYPTO_GROUP_MAX);
+		ret = -EINVAL;
+		goto unlock;
+	}
+
+	hdr_len = NX842_CRYPTO_HEADER_SIZE(hdr->groups);
+	if (hdr_len > slen) {
+		ret = -EOVERFLOW;
+		goto unlock;
+	}
+
+	memcpy(&ctx->header, src, hdr_len);
+	hdr = container_of(&ctx->header, struct nx842_crypto_header, hdr);
+
+	for (n = 0; n < hdr->groups; n++) {
+		/* ignore applies to last group */
+		if (n + 1 == hdr->groups)
+			ignore = be16_to_cpu(hdr->ignore);
+
+		ret = decompress(ctx, &p, &hdr->group[n], &c, ignore);
+		if (ret)
+			goto unlock;
+	}
+
+success:
+	*dlen = p.ototal;
+
+	pr_debug("decompress total slen %x dlen %x\n", slen, *dlen);
+
+	ret = 0;
+
+unlock:
+	spin_unlock_bh(&ctx->lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(nx842_crypto_decompress);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("IBM PowerPC Nest (NX) 842 Hardware Compression Driver");
+MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");

linux-7.2.3/drivers/crypto/nx/nx-842.h

@@ -0,0 +1,195 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __NX_842_H__
+#define __NX_842_H__
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/ratelimit.h>
+
+/* Restrictions on Data Descriptor List (DDL) and Entry (DDE) buffers
+ *
+ * From NX P8 workbook, sec 4.9.1 "842 details"
+ *   Each DDE buffer is 128 byte aligned
+ *   Each DDE buffer size is a multiple of 32 bytes (except the last)
+ *   The last DDE buffer size is a multiple of 8 bytes
+ */
+#define DDE_BUFFER_ALIGN	(128)
+#define DDE_BUFFER_SIZE_MULT	(32)
+#define DDE_BUFFER_LAST_MULT	(8)
+
+/* Arbitrary DDL length limit
+ * Allows max buffer size of MAX-1 to MAX pages
+ * (depending on alignment)
+ */
+#define DDL_LEN_MAX		(17)
+
+/* CCW 842 CI/FC masks
+ * NX P8 workbook, section 4.3.1, figure 4-6
+ * "CI/FC Boundary by NX CT type"
+ */
+#define CCW_CI_842		(0x00003ff8)
+#define CCW_FC_842		(0x00000007)
+
+/* CCW Function Codes (FC) for 842
+ * NX P8 workbook, section 4.9, table 4-28
+ * "Function Code Definitions for 842 Memory Compression"
+ */
+#define CCW_FC_842_COMP_NOCRC	(0)
+#define CCW_FC_842_COMP_CRC	(1)
+#define CCW_FC_842_DECOMP_NOCRC	(2)
+#define CCW_FC_842_DECOMP_CRC	(3)
+#define CCW_FC_842_MOVE		(4)
+
+/* CSB CC Error Types for 842
+ * NX P8 workbook, section 4.10.3, table 4-30
+ * "Reported Error Types Summary Table"
+ */
+/* These are all duplicates of existing codes defined in icswx.h. */
+#define CSB_CC_TRANSLATION_DUP1	(80)
+#define CSB_CC_TRANSLATION_DUP2	(82)
+#define CSB_CC_TRANSLATION_DUP3	(84)
+#define CSB_CC_TRANSLATION_DUP4	(86)
+#define CSB_CC_TRANSLATION_DUP5	(92)
+#define CSB_CC_TRANSLATION_DUP6	(94)
+#define CSB_CC_PROTECTION_DUP1	(81)
+#define CSB_CC_PROTECTION_DUP2	(83)
+#define CSB_CC_PROTECTION_DUP3	(85)
+#define CSB_CC_PROTECTION_DUP4	(87)
+#define CSB_CC_PROTECTION_DUP5	(93)
+#define CSB_CC_PROTECTION_DUP6	(95)
+#define CSB_CC_RD_EXTERNAL_DUP1	(89)
+#define CSB_CC_RD_EXTERNAL_DUP2	(90)
+#define CSB_CC_RD_EXTERNAL_DUP3	(91)
+/* These are specific to NX */
+/* 842 codes */
+#define CSB_CC_TPBC_GT_SPBC	(64) /* no error, but >1 comp ratio */
+#define CSB_CC_CRC_MISMATCH	(65) /* decomp crc mismatch */
+#define CSB_CC_TEMPL_INVALID	(66) /* decomp invalid template value */
+#define CSB_CC_TEMPL_OVERFLOW	(67) /* decomp template shows data after end */
+/* sym crypt codes */
+#define CSB_CC_DECRYPT_OVERFLOW	(64)
+/* asym crypt codes */
+#define CSB_CC_MINV_OVERFLOW	(128)
+/*
+ * HW error - Job did not finish in the maximum time allowed.
+ * Job terminated.
+ */
+#define CSB_CC_HW_EXPIRED_TIMER		(224)
+/* These are reserved for hypervisor use */
+#define CSB_CC_HYP_RESERVE_START	(240)
+#define CSB_CC_HYP_RESERVE_END		(253)
+#define CSB_CC_HYP_RESERVE_P9_END	(251)
+/* No valid interrupt server (P9 or later). */
+#define CSB_CC_HYP_RESERVE_NO_INTR_SERVER	(252)
+#define CSB_CC_HYP_NO_HW		(254)
+#define CSB_CC_HYP_HANG_ABORTED		(255)
+
+/* CCB Completion Modes (CM) for 842
+ * NX P8 workbook, section 4.3, figure 4-5
+ * "CRB Details - Normal Cop_Req (CL=00, C=1)"
+ */
+#define CCB_CM_EXTRA_WRITE	(CCB_CM0_ALL_COMPLETIONS & CCB_CM12_STORE)
+#define CCB_CM_INTERRUPT	(CCB_CM0_ALL_COMPLETIONS & CCB_CM12_INTERRUPT)
+
+#define LEN_ON_SIZE(pa, size)	((size) - ((pa) & ((size) - 1)))
+#define LEN_ON_PAGE(pa)		LEN_ON_SIZE(pa, PAGE_SIZE)
+
+struct crypto_scomp;
+
+static inline unsigned long nx842_get_pa(void *addr)
+{
+	if (!is_vmalloc_addr(addr))
+		return __pa(addr);
+
+	return page_to_phys(vmalloc_to_page(addr)) + offset_in_page(addr);
+}
+
+/**
+ * This provides the driver's constraints.  Different nx842 implementations
+ * may have varying requirements.  The constraints are:
+ *   @alignment:	All buffers should be aligned to this
+ *   @multiple:		All buffer lengths should be a multiple of this
+ *   @minimum:		Buffer lengths must not be less than this amount
+ *   @maximum:		Buffer lengths must not be more than this amount
+ *
+ * The constraints apply to all buffers and lengths, both input and output,
+ * for both compression and decompression, except for the minimum which
+ * only applies to compression input and decompression output; the
+ * compressed data can be less than the minimum constraint.  It can be
+ * assumed that compressed data will always adhere to the multiple
+ * constraint.
+ *
+ * The driver may succeed even if these constraints are violated;
+ * however the driver can return failure or suffer reduced performance
+ * if any constraint is not met.
+ */
+struct nx842_constraints {
+	int alignment;
+	int multiple;
+	int minimum;
+	int maximum;
+};
+
+struct nx842_driver {
+	char *name;
+	struct module *owner;
+	size_t workmem_size;
+
+	struct nx842_constraints *constraints;
+
+	int (*compress)(const unsigned char *in, unsigned int in_len,
+			unsigned char *out, unsigned int *out_len,
+			void *wrkmem);
+	int (*decompress)(const unsigned char *in, unsigned int in_len,
+			  unsigned char *out, unsigned int *out_len,
+			  void *wrkmem);
+};
+
+struct nx842_crypto_header_group {
+	__be16 padding;			/* unused bytes at start of group */
+	__be32 compressed_length;	/* compressed bytes in group */
+	__be32 uncompressed_length;	/* bytes after decompression */
+} __packed;
+
+struct nx842_crypto_header {
+	/* New members MUST be added within the struct_group() macro below. */
+	__struct_group(nx842_crypto_header_hdr, hdr, __packed,
+		__be16 magic;		/* NX842_CRYPTO_MAGIC */
+		__be16 ignore;		/* decompressed end bytes to ignore */
+		u8 groups;		/* total groups in this header */
+	);
+	struct nx842_crypto_header_group group[] __counted_by(groups);
+} __packed;
+static_assert(offsetof(struct nx842_crypto_header, group) == sizeof(struct nx842_crypto_header_hdr),
+	      "struct member likely outside of __struct_group()");
+
+#define NX842_CRYPTO_GROUP_MAX	(0x20)
+
+struct nx842_crypto_ctx {
+	spinlock_t lock;
+
+	u8 *wmem;
+	u8 *sbounce, *dbounce;
+
+	struct nx842_crypto_header_hdr header;
+	struct nx842_crypto_header_group group[NX842_CRYPTO_GROUP_MAX];
+
+	struct nx842_driver *driver;
+};
+
+void *nx842_crypto_alloc_ctx(struct nx842_driver *driver);
+void nx842_crypto_free_ctx(void *ctx);
+int nx842_crypto_compress(struct crypto_scomp *tfm,
+			  const u8 *src, unsigned int slen,
+			  u8 *dst, unsigned int *dlen, void *ctx);
+int nx842_crypto_decompress(struct crypto_scomp *tfm,
+			    const u8 *src, unsigned int slen,
+			    u8 *dst, unsigned int *dlen, void *ctx);
+
+#endif /* __NX_842_H__ */

linux-7.2.3/drivers/crypto/nx/nx-aes-cbc.c

@@ -0,0 +1,129 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AES CBC routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2011-2012 International Business Machines Inc.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/aes.h>
+#include <crypto/internal/skcipher.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+
+static int cbc_aes_nx_set_key(struct crypto_skcipher *tfm,
+			      const u8               *in_key,
+			      unsigned int            key_len)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm);
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+
+	nx_ctx_init(nx_ctx, HCOP_FC_AES);
+
+	switch (key_len) {
+	case AES_KEYSIZE_128:
+		NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
+		nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
+		break;
+	case AES_KEYSIZE_192:
+		NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_192);
+		nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_192];
+		break;
+	case AES_KEYSIZE_256:
+		NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_256);
+		nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_256];
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	csbcpb->cpb.hdr.mode = NX_MODE_AES_CBC;
+	memcpy(csbcpb->cpb.aes_cbc.key, in_key, key_len);
+
+	return 0;
+}
+
+static int cbc_aes_nx_crypt(struct skcipher_request *req,
+			    int                      enc)
+{
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm);
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	unsigned long irq_flags;
+	unsigned int processed = 0, to_process;
+	int rc;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	if (enc)
+		NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+	else
+		NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+
+	do {
+		to_process = req->cryptlen - processed;
+
+		rc = nx_build_sg_lists(nx_ctx, req->iv, req->dst, req->src,
+				       &to_process, processed,
+				       csbcpb->cpb.aes_cbc.iv);
+		if (rc)
+			goto out;
+
+		if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+			rc = -EINVAL;
+			goto out;
+		}
+
+		rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+				   req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+		if (rc)
+			goto out;
+
+		memcpy(req->iv, csbcpb->cpb.aes_cbc.cv, AES_BLOCK_SIZE);
+		atomic_inc(&(nx_ctx->stats->aes_ops));
+		atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count),
+			     &(nx_ctx->stats->aes_bytes));
+
+		processed += to_process;
+	} while (processed < req->cryptlen);
+out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+	return rc;
+}
+
+static int cbc_aes_nx_encrypt(struct skcipher_request *req)
+{
+	return cbc_aes_nx_crypt(req, 1);
+}
+
+static int cbc_aes_nx_decrypt(struct skcipher_request *req)
+{
+	return cbc_aes_nx_crypt(req, 0);
+}
+
+struct skcipher_alg nx_cbc_aes_alg = {
+	.base.cra_name		= "cbc(aes)",
+	.base.cra_driver_name	= "cbc-aes-nx",
+	.base.cra_priority	= 300,
+	.base.cra_blocksize	= AES_BLOCK_SIZE,
+	.base.cra_ctxsize	= sizeof(struct nx_crypto_ctx),
+	.base.cra_alignmask	= 0xf,
+	.base.cra_module	= THIS_MODULE,
+	.init			= nx_crypto_ctx_aes_cbc_init,
+	.exit			= nx_crypto_ctx_skcipher_exit,
+	.min_keysize		= AES_MIN_KEY_SIZE,
+	.max_keysize		= AES_MAX_KEY_SIZE,
+	.ivsize			= AES_BLOCK_SIZE,
+	.setkey			= cbc_aes_nx_set_key,
+	.encrypt		= cbc_aes_nx_encrypt,
+	.decrypt		= cbc_aes_nx_decrypt,
+};

linux-7.2.3/drivers/crypto/nx/nx-aes-ccm.c

@@ -0,0 +1,559 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AES CCM routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2012 International Business Machines Inc.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/internal/aead.h>
+#include <crypto/aes.h>
+#include <crypto/algapi.h>
+#include <crypto/scatterwalk.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/crypto.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+
+static int ccm_aes_nx_set_key(struct crypto_aead *tfm,
+			      const u8           *in_key,
+			      unsigned int        key_len)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead;
+
+	nx_ctx_init(nx_ctx, HCOP_FC_AES);
+
+	switch (key_len) {
+	case AES_KEYSIZE_128:
+		NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
+		NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_128);
+		nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	csbcpb->cpb.hdr.mode = NX_MODE_AES_CCM;
+	memcpy(csbcpb->cpb.aes_ccm.key, in_key, key_len);
+
+	csbcpb_aead->cpb.hdr.mode = NX_MODE_AES_CCA;
+	memcpy(csbcpb_aead->cpb.aes_cca.key, in_key, key_len);
+
+	return 0;
+
+}
+
+static int ccm4309_aes_nx_set_key(struct crypto_aead *tfm,
+				  const u8           *in_key,
+				  unsigned int        key_len)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);
+
+	if (key_len < 3)
+		return -EINVAL;
+
+	key_len -= 3;
+
+	memcpy(nx_ctx->priv.ccm.nonce, in_key + key_len, 3);
+
+	return ccm_aes_nx_set_key(tfm, in_key, key_len);
+}
+
+static int ccm_aes_nx_setauthsize(struct crypto_aead *tfm,
+				  unsigned int authsize)
+{
+	switch (authsize) {
+	case 4:
+	case 6:
+	case 8:
+	case 10:
+	case 12:
+	case 14:
+	case 16:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int ccm4309_aes_nx_setauthsize(struct crypto_aead *tfm,
+				      unsigned int authsize)
+{
+	switch (authsize) {
+	case 8:
+	case 12:
+	case 16:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/* taken from crypto/ccm.c */
+static int set_msg_len(u8 *block, unsigned int msglen, int csize)
+{
+	__be32 data;
+
+	memset(block, 0, csize);
+	block += csize;
+
+	if (csize >= 4)
+		csize = 4;
+	else if (msglen > (unsigned int)(1 << (8 * csize)))
+		return -EOVERFLOW;
+
+	data = cpu_to_be32(msglen);
+	memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
+
+	return 0;
+}
+
+/* taken from crypto/ccm.c */
+static inline int crypto_ccm_check_iv(const u8 *iv)
+{
+	/* 2 <= L <= 8, so 1 <= L' <= 7. */
+	if (1 > iv[0] || iv[0] > 7)
+		return -EINVAL;
+
+	return 0;
+}
+
+/* based on code from crypto/ccm.c */
+static int generate_b0(u8 *iv, unsigned int assoclen, unsigned int authsize,
+		       unsigned int cryptlen, u8 *b0)
+{
+	unsigned int l, lp, m = authsize;
+
+	memcpy(b0, iv, 16);
+
+	lp = b0[0];
+	l = lp + 1;
+
+	/* set m, bits 3-5 */
+	*b0 |= (8 * ((m - 2) / 2));
+
+	/* set adata, bit 6, if associated data is used */
+	if (assoclen)
+		*b0 |= 64;
+
+	return set_msg_len(b0 + 16 - l, cryptlen, l);
+}
+
+static int generate_pat(u8                   *iv,
+			struct aead_request  *req,
+			struct nx_crypto_ctx *nx_ctx,
+			unsigned int          authsize,
+			unsigned int          nbytes,
+			unsigned int	      assoclen,
+			u8                   *out)
+{
+	struct nx_sg *nx_insg = nx_ctx->in_sg;
+	struct nx_sg *nx_outsg = nx_ctx->out_sg;
+	unsigned int iauth_len = 0;
+	u8 tmp[16], *b1 = NULL, *b0 = NULL, *result = NULL;
+	int rc;
+	unsigned int max_sg_len;
+
+	/* zero the ctr value */
+	memset(iv + 15 - iv[0], 0, iv[0] + 1);
+
+	/* page 78 of nx_wb.pdf has,
+	 * Note: RFC3610 allows the AAD data to be up to 2^64 -1 bytes
+	 * in length. If a full message is used, the AES CCA implementation
+	 * restricts the maximum AAD length to 2^32 -1 bytes.
+	 * If partial messages are used, the implementation supports
+	 * 2^64 -1 bytes maximum AAD length.
+	 *
+	 * However, in the cryptoapi's aead_request structure,
+	 * assoclen is an unsigned int, thus it cannot hold a length
+	 * value greater than 2^32 - 1.
+	 * Thus the AAD is further constrained by this and is never
+	 * greater than 2^32.
+	 */
+
+	if (!assoclen) {
+		b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;
+	} else if (assoclen <= 14) {
+		/* if associated data is 14 bytes or less, we do 1 GCM
+		 * operation on 2 AES blocks, B0 (stored in the csbcpb) and B1,
+		 * which is fed in through the source buffers here */
+		b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;
+		b1 = nx_ctx->priv.ccm.iauth_tag;
+		iauth_len = assoclen;
+	} else if (assoclen <= 65280) {
+		/* if associated data is less than (2^16 - 2^8), we construct
+		 * B1 differently and feed in the associated data to a CCA
+		 * operation */
+		b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;
+		b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;
+		iauth_len = 14;
+	} else {
+		b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;
+		b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;
+		iauth_len = 10;
+	}
+
+	/* generate B0 */
+	rc = generate_b0(iv, assoclen, authsize, nbytes, b0);
+	if (rc)
+		return rc;
+
+	/* generate B1:
+	 * add control info for associated data
+	 * RFC 3610 and NIST Special Publication 800-38C
+	 */
+	if (b1) {
+		memset(b1, 0, 16);
+		if (assoclen <= 65280) {
+			*(u16 *)b1 = assoclen;
+			memcpy_from_sglist(b1 + 2, req->src, 0, iauth_len);
+		} else {
+			*(u16 *)b1 = (u16)(0xfffe);
+			*(u32 *)&b1[2] = assoclen;
+			memcpy_from_sglist(b1 + 6, req->src, 0, iauth_len);
+		}
+	}
+
+	/* now copy any remaining AAD to scatterlist and call nx... */
+	if (!assoclen) {
+		return rc;
+	} else if (assoclen <= 14) {
+		unsigned int len = 16;
+
+		nx_insg = nx_build_sg_list(nx_insg, b1, &len, nx_ctx->ap->sglen);
+
+		if (len != 16)
+			return -EINVAL;
+
+		nx_outsg = nx_build_sg_list(nx_outsg, tmp, &len,
+					    nx_ctx->ap->sglen);
+
+		if (len != 16)
+			return -EINVAL;
+
+		/* inlen should be negative, indicating to phyp that its a
+		 * pointer to an sg list */
+		nx_ctx->op.inlen = (nx_ctx->in_sg - nx_insg) *
+					sizeof(struct nx_sg);
+		nx_ctx->op.outlen = (nx_ctx->out_sg - nx_outsg) *
+					sizeof(struct nx_sg);
+
+		NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+		NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_INTERMEDIATE;
+
+		result = nx_ctx->csbcpb->cpb.aes_ccm.out_pat_or_mac;
+
+		rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+				   req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+		if (rc)
+			return rc;
+
+		atomic_inc(&(nx_ctx->stats->aes_ops));
+		atomic64_add(assoclen, &nx_ctx->stats->aes_bytes);
+
+	} else {
+		unsigned int processed = 0, to_process;
+
+		processed += iauth_len;
+
+		/* page_limit: number of sg entries that fit on one page */
+		max_sg_len = min_t(u64, nx_ctx->ap->sglen,
+				nx_driver.of.max_sg_len/sizeof(struct nx_sg));
+		max_sg_len = min_t(u64, max_sg_len,
+				nx_ctx->ap->databytelen/NX_PAGE_SIZE);
+
+		do {
+			to_process = min_t(u32, assoclen - processed,
+					   nx_ctx->ap->databytelen);
+
+			nx_insg = nx_walk_and_build(nx_ctx->in_sg,
+						    nx_ctx->ap->sglen,
+						    req->src, processed,
+						    &to_process);
+
+			if ((to_process + processed) < assoclen) {
+				NX_CPB_FDM(nx_ctx->csbcpb_aead) |=
+					NX_FDM_INTERMEDIATE;
+			} else {
+				NX_CPB_FDM(nx_ctx->csbcpb_aead) &=
+					~NX_FDM_INTERMEDIATE;
+			}
+
+
+			nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_insg) *
+						sizeof(struct nx_sg);
+
+			result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
+
+			rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead,
+				   req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+			if (rc)
+				return rc;
+
+			memcpy(nx_ctx->csbcpb_aead->cpb.aes_cca.b0,
+				nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0,
+				AES_BLOCK_SIZE);
+
+			NX_CPB_FDM(nx_ctx->csbcpb_aead) |= NX_FDM_CONTINUATION;
+
+			atomic_inc(&(nx_ctx->stats->aes_ops));
+			atomic64_add(assoclen, &nx_ctx->stats->aes_bytes);
+
+			processed += to_process;
+		} while (processed < assoclen);
+
+		result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
+	}
+
+	memcpy(out, result, AES_BLOCK_SIZE);
+
+	return rc;
+}
+
+static int ccm_nx_decrypt(struct aead_request   *req,
+			  u8                    *iv,
+			  unsigned int assoclen)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	unsigned int nbytes = req->cryptlen;
+	unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
+	struct nx_ccm_priv *priv = &nx_ctx->priv.ccm;
+	unsigned long irq_flags;
+	unsigned int processed = 0, to_process;
+	int rc = -1;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	nbytes -= authsize;
+
+	/* copy out the auth tag to compare with later */
+	memcpy_from_sglist(priv->oauth_tag, req->src, nbytes + req->assoclen,
+			   authsize);
+
+	rc = generate_pat(iv, req, nx_ctx, authsize, nbytes, assoclen,
+			  csbcpb->cpb.aes_ccm.in_pat_or_b0);
+	if (rc)
+		goto out;
+
+	do {
+
+		/* to_process: the AES_BLOCK_SIZE data chunk to process in this
+		 * update. This value is bound by sg list limits.
+		 */
+		to_process = nbytes - processed;
+
+		if ((to_process + processed) < nbytes)
+			NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+		else
+			NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+		NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+
+		rc = nx_build_sg_lists(nx_ctx, iv, req->dst, req->src,
+				       &to_process, processed + req->assoclen,
+				       csbcpb->cpb.aes_ccm.iv_or_ctr);
+		if (rc)
+			goto out;
+
+		rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+			   req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+		if (rc)
+			goto out;
+
+		/* for partial completion, copy following for next
+		 * entry into loop...
+		 */
+		memcpy(iv, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE);
+		memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0,
+			csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE);
+		memcpy(csbcpb->cpb.aes_ccm.in_s0,
+			csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE);
+
+		NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+		/* update stats */
+		atomic_inc(&(nx_ctx->stats->aes_ops));
+		atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count),
+			     &(nx_ctx->stats->aes_bytes));
+
+		processed += to_process;
+	} while (processed < nbytes);
+
+	rc = crypto_memneq(csbcpb->cpb.aes_ccm.out_pat_or_mac, priv->oauth_tag,
+		    authsize) ? -EBADMSG : 0;
+out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+	return rc;
+}
+
+static int ccm_nx_encrypt(struct aead_request   *req,
+			  u8                    *iv,
+			  unsigned int assoclen)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	unsigned int nbytes = req->cryptlen;
+	unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
+	unsigned long irq_flags;
+	unsigned int processed = 0, to_process;
+	int rc = -1;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	rc = generate_pat(iv, req, nx_ctx, authsize, nbytes, assoclen,
+			  csbcpb->cpb.aes_ccm.in_pat_or_b0);
+	if (rc)
+		goto out;
+
+	do {
+		/* to process: the AES_BLOCK_SIZE data chunk to process in this
+		 * update. This value is bound by sg list limits.
+		 */
+		to_process = nbytes - processed;
+
+		if ((to_process + processed) < nbytes)
+			NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+		else
+			NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+		NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+
+		rc = nx_build_sg_lists(nx_ctx, iv, req->dst, req->src,
+				       &to_process, processed + req->assoclen,
+				       csbcpb->cpb.aes_ccm.iv_or_ctr);
+		if (rc)
+			goto out;
+
+		rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+				   req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+		if (rc)
+			goto out;
+
+		/* for partial completion, copy following for next
+		 * entry into loop...
+		 */
+		memcpy(iv, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE);
+		memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0,
+			csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE);
+		memcpy(csbcpb->cpb.aes_ccm.in_s0,
+			csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE);
+
+		NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+		/* update stats */
+		atomic_inc(&(nx_ctx->stats->aes_ops));
+		atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count),
+			     &(nx_ctx->stats->aes_bytes));
+
+		processed += to_process;
+
+	} while (processed < nbytes);
+
+	/* copy out the auth tag */
+	memcpy_to_sglist(req->dst, nbytes + req->assoclen,
+			 csbcpb->cpb.aes_ccm.out_pat_or_mac, authsize);
+
+out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+	return rc;
+}
+
+static int ccm4309_aes_nx_encrypt(struct aead_request *req)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+	struct nx_gcm_rctx *rctx = aead_request_ctx(req);
+	u8 *iv = rctx->iv;
+
+	iv[0] = 3;
+	memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3);
+	memcpy(iv + 4, req->iv, 8);
+
+	return ccm_nx_encrypt(req, iv, req->assoclen - 8);
+}
+
+static int ccm_aes_nx_encrypt(struct aead_request *req)
+{
+	int rc;
+
+	rc = crypto_ccm_check_iv(req->iv);
+	if (rc)
+		return rc;
+
+	return ccm_nx_encrypt(req, req->iv, req->assoclen);
+}
+
+static int ccm4309_aes_nx_decrypt(struct aead_request *req)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+	struct nx_gcm_rctx *rctx = aead_request_ctx(req);
+	u8 *iv = rctx->iv;
+
+	iv[0] = 3;
+	memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3);
+	memcpy(iv + 4, req->iv, 8);
+
+	return ccm_nx_decrypt(req, iv, req->assoclen - 8);
+}
+
+static int ccm_aes_nx_decrypt(struct aead_request *req)
+{
+	int rc;
+
+	rc = crypto_ccm_check_iv(req->iv);
+	if (rc)
+		return rc;
+
+	return ccm_nx_decrypt(req, req->iv, req->assoclen);
+}
+
+struct aead_alg nx_ccm_aes_alg = {
+	.base = {
+		.cra_name        = "ccm(aes)",
+		.cra_driver_name = "ccm-aes-nx",
+		.cra_priority    = 300,
+		.cra_flags       = CRYPTO_ALG_NEED_FALLBACK,
+		.cra_blocksize   = 1,
+		.cra_ctxsize     = sizeof(struct nx_crypto_ctx),
+		.cra_module      = THIS_MODULE,
+	},
+	.init        = nx_crypto_ctx_aes_ccm_init,
+	.exit        = nx_crypto_ctx_aead_exit,
+	.ivsize      = AES_BLOCK_SIZE,
+	.maxauthsize = AES_BLOCK_SIZE,
+	.setkey      = ccm_aes_nx_set_key,
+	.setauthsize = ccm_aes_nx_setauthsize,
+	.encrypt     = ccm_aes_nx_encrypt,
+	.decrypt     = ccm_aes_nx_decrypt,
+};
+
+struct aead_alg nx_ccm4309_aes_alg = {
+	.base = {
+		.cra_name        = "rfc4309(ccm(aes))",
+		.cra_driver_name = "rfc4309-ccm-aes-nx",
+		.cra_priority    = 300,
+		.cra_flags       = CRYPTO_ALG_NEED_FALLBACK,
+		.cra_blocksize   = 1,
+		.cra_ctxsize     = sizeof(struct nx_crypto_ctx),
+		.cra_module      = THIS_MODULE,
+	},
+	.init        = nx_crypto_ctx_aes_ccm_init,
+	.exit        = nx_crypto_ctx_aead_exit,
+	.ivsize      = 8,
+	.maxauthsize = AES_BLOCK_SIZE,
+	.setkey      = ccm4309_aes_nx_set_key,
+	.setauthsize = ccm4309_aes_nx_setauthsize,
+	.encrypt     = ccm4309_aes_nx_encrypt,
+	.decrypt     = ccm4309_aes_nx_decrypt,
+};

linux-7.2.3/drivers/crypto/nx/nx-aes-ctr.c

@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AES CTR routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2011-2012 International Business Machines Inc.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/aes.h>
+#include <crypto/ctr.h>
+#include <crypto/internal/skcipher.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+
+static int ctr_aes_nx_set_key(struct crypto_skcipher *tfm,
+			      const u8               *in_key,
+			      unsigned int            key_len)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm);
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+
+	nx_ctx_init(nx_ctx, HCOP_FC_AES);
+
+	switch (key_len) {
+	case AES_KEYSIZE_128:
+		NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
+		nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
+		break;
+	case AES_KEYSIZE_192:
+		NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_192);
+		nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_192];
+		break;
+	case AES_KEYSIZE_256:
+		NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_256);
+		nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_256];
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	csbcpb->cpb.hdr.mode = NX_MODE_AES_CTR;
+	memcpy(csbcpb->cpb.aes_ctr.key, in_key, key_len);
+
+	return 0;
+}
+
+static int ctr3686_aes_nx_set_key(struct crypto_skcipher *tfm,
+				  const u8               *in_key,
+				  unsigned int            key_len)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm);
+
+	if (key_len < CTR_RFC3686_NONCE_SIZE)
+		return -EINVAL;
+
+	memcpy(nx_ctx->priv.ctr.nonce,
+	       in_key + key_len - CTR_RFC3686_NONCE_SIZE,
+	       CTR_RFC3686_NONCE_SIZE);
+
+	key_len -= CTR_RFC3686_NONCE_SIZE;
+
+	return ctr_aes_nx_set_key(tfm, in_key, key_len);
+}
+
+static int ctr_aes_nx_crypt(struct skcipher_request *req, u8 *iv)
+{
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm);
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	unsigned long irq_flags;
+	unsigned int processed = 0, to_process;
+	int rc;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	do {
+		to_process = req->cryptlen - processed;
+
+		rc = nx_build_sg_lists(nx_ctx, iv, req->dst, req->src,
+				       &to_process, processed,
+				       csbcpb->cpb.aes_ctr.iv);
+		if (rc)
+			goto out;
+
+		if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+			rc = -EINVAL;
+			goto out;
+		}
+
+		rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+				   req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+		if (rc)
+			goto out;
+
+		memcpy(iv, csbcpb->cpb.aes_cbc.cv, AES_BLOCK_SIZE);
+
+		atomic_inc(&(nx_ctx->stats->aes_ops));
+		atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count),
+			     &(nx_ctx->stats->aes_bytes));
+
+		processed += to_process;
+	} while (processed < req->cryptlen);
+out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+	return rc;
+}
+
+static int ctr3686_aes_nx_crypt(struct skcipher_request *req)
+{
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm);
+	u8 iv[16];
+
+	memcpy(iv, nx_ctx->priv.ctr.nonce, CTR_RFC3686_NONCE_SIZE);
+	memcpy(iv + CTR_RFC3686_NONCE_SIZE, req->iv, CTR_RFC3686_IV_SIZE);
+	iv[12] = iv[13] = iv[14] = 0;
+	iv[15] = 1;
+
+	return ctr_aes_nx_crypt(req, iv);
+}
+
+struct skcipher_alg nx_ctr3686_aes_alg = {
+	.base.cra_name		= "rfc3686(ctr(aes))",
+	.base.cra_driver_name	= "rfc3686-ctr-aes-nx",
+	.base.cra_priority	= 300,
+	.base.cra_blocksize	= 1,
+	.base.cra_ctxsize	= sizeof(struct nx_crypto_ctx),
+	.base.cra_module	= THIS_MODULE,
+	.init			= nx_crypto_ctx_aes_ctr_init,
+	.exit			= nx_crypto_ctx_skcipher_exit,
+	.min_keysize		= AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
+	.max_keysize		= AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
+	.ivsize			= CTR_RFC3686_IV_SIZE,
+	.setkey			= ctr3686_aes_nx_set_key,
+	.encrypt		= ctr3686_aes_nx_crypt,
+	.decrypt		= ctr3686_aes_nx_crypt,
+	.chunksize		= AES_BLOCK_SIZE,
+};

linux-7.2.3/drivers/crypto/nx/nx-aes-ecb.c

@@ -0,0 +1,127 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AES ECB routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2011-2012 International Business Machines Inc.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/aes.h>
+#include <crypto/internal/skcipher.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+
+static int ecb_aes_nx_set_key(struct crypto_skcipher *tfm,
+			      const u8               *in_key,
+			      unsigned int            key_len)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm);
+	struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
+
+	nx_ctx_init(nx_ctx, HCOP_FC_AES);
+
+	switch (key_len) {
+	case AES_KEYSIZE_128:
+		NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
+		nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
+		break;
+	case AES_KEYSIZE_192:
+		NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_192);
+		nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_192];
+		break;
+	case AES_KEYSIZE_256:
+		NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_256);
+		nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_256];
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	csbcpb->cpb.hdr.mode = NX_MODE_AES_ECB;
+	memcpy(csbcpb->cpb.aes_ecb.key, in_key, key_len);
+
+	return 0;
+}
+
+static int ecb_aes_nx_crypt(struct skcipher_request *req,
+			    int                      enc)
+{
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm);
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	unsigned long irq_flags;
+	unsigned int processed = 0, to_process;
+	int rc;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	if (enc)
+		NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+	else
+		NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+
+	do {
+		to_process = req->cryptlen - processed;
+
+		rc = nx_build_sg_lists(nx_ctx, NULL, req->dst, req->src,
+				       &to_process, processed, NULL);
+		if (rc)
+			goto out;
+
+		if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+			rc = -EINVAL;
+			goto out;
+		}
+
+		rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+				   req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+		if (rc)
+			goto out;
+
+		atomic_inc(&(nx_ctx->stats->aes_ops));
+		atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count),
+			     &(nx_ctx->stats->aes_bytes));
+
+		processed += to_process;
+	} while (processed < req->cryptlen);
+
+out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+	return rc;
+}
+
+static int ecb_aes_nx_encrypt(struct skcipher_request *req)
+{
+	return ecb_aes_nx_crypt(req, 1);
+}
+
+static int ecb_aes_nx_decrypt(struct skcipher_request *req)
+{
+	return ecb_aes_nx_crypt(req, 0);
+}
+
+struct skcipher_alg nx_ecb_aes_alg = {
+	.base.cra_name		= "ecb(aes)",
+	.base.cra_driver_name	= "ecb-aes-nx",
+	.base.cra_priority	= 300,
+	.base.cra_blocksize	= AES_BLOCK_SIZE,
+	.base.cra_alignmask	= 0xf,
+	.base.cra_ctxsize	= sizeof(struct nx_crypto_ctx),
+	.base.cra_module	= THIS_MODULE,
+	.init			= nx_crypto_ctx_aes_ecb_init,
+	.exit			= nx_crypto_ctx_skcipher_exit,
+	.min_keysize		= AES_MIN_KEY_SIZE,
+	.max_keysize		= AES_MAX_KEY_SIZE,
+	.setkey			= ecb_aes_nx_set_key,
+	.encrypt		= ecb_aes_nx_encrypt,
+	.decrypt		= ecb_aes_nx_decrypt,
+};

linux-7.2.3/drivers/crypto/nx/nx-aes-gcm.c

@@ -0,0 +1,500 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AES GCM routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2012 International Business Machines Inc.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/internal/aead.h>
+#include <crypto/aes.h>
+#include <crypto/algapi.h>
+#include <crypto/gcm.h>
+#include <crypto/scatterwalk.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+
+static int gcm_aes_nx_set_key(struct crypto_aead *tfm,
+			      const u8           *in_key,
+			      unsigned int        key_len)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_aead_ctx(tfm);
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead;
+
+	nx_ctx_init(nx_ctx, HCOP_FC_AES);
+
+	switch (key_len) {
+	case AES_KEYSIZE_128:
+		NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
+		NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_128);
+		nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
+		break;
+	case AES_KEYSIZE_192:
+		NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_192);
+		NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_192);
+		nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_192];
+		break;
+	case AES_KEYSIZE_256:
+		NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_256);
+		NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_256);
+		nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_256];
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	csbcpb->cpb.hdr.mode = NX_MODE_AES_GCM;
+	memcpy(csbcpb->cpb.aes_gcm.key, in_key, key_len);
+
+	csbcpb_aead->cpb.hdr.mode = NX_MODE_AES_GCA;
+	memcpy(csbcpb_aead->cpb.aes_gca.key, in_key, key_len);
+
+	return 0;
+}
+
+static int gcm4106_aes_nx_set_key(struct crypto_aead *tfm,
+				  const u8           *in_key,
+				  unsigned int        key_len)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_aead_ctx(tfm);
+	char *nonce = nx_ctx->priv.gcm.nonce;
+	int rc;
+
+	if (key_len < 4)
+		return -EINVAL;
+
+	key_len -= 4;
+
+	rc = gcm_aes_nx_set_key(tfm, in_key, key_len);
+	if (rc)
+		goto out;
+
+	memcpy(nonce, in_key + key_len, 4);
+out:
+	return rc;
+}
+
+static int gcm4106_aes_nx_setauthsize(struct crypto_aead *tfm,
+				      unsigned int authsize)
+{
+	switch (authsize) {
+	case 8:
+	case 12:
+	case 16:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int nx_gca(struct nx_crypto_ctx  *nx_ctx,
+		  struct aead_request   *req,
+		  u8                    *out,
+		  unsigned int assoclen)
+{
+	int rc;
+	struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead;
+	struct nx_sg *nx_sg = nx_ctx->in_sg;
+	unsigned int nbytes = assoclen;
+	unsigned int processed = 0, to_process;
+	unsigned int max_sg_len;
+
+	if (nbytes <= AES_BLOCK_SIZE) {
+		memcpy_from_sglist(out, req->src, 0, nbytes);
+		return 0;
+	}
+
+	NX_CPB_FDM(csbcpb_aead) &= ~NX_FDM_CONTINUATION;
+
+	/* page_limit: number of sg entries that fit on one page */
+	max_sg_len = min_t(u64, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+			   nx_ctx->ap->sglen);
+	max_sg_len = min_t(u64, max_sg_len,
+			   nx_ctx->ap->databytelen/NX_PAGE_SIZE);
+
+	do {
+		/*
+		 * to_process: the data chunk to process in this update.
+		 * This value is bound by sg list limits.
+		 */
+		to_process = min_t(u64, nbytes - processed,
+				   nx_ctx->ap->databytelen);
+		to_process = min_t(u64, to_process,
+				   NX_PAGE_SIZE * (max_sg_len - 1));
+
+		nx_sg = nx_walk_and_build(nx_ctx->in_sg, max_sg_len,
+					  req->src, processed, &to_process);
+
+		if ((to_process + processed) < nbytes)
+			NX_CPB_FDM(csbcpb_aead) |= NX_FDM_INTERMEDIATE;
+		else
+			NX_CPB_FDM(csbcpb_aead) &= ~NX_FDM_INTERMEDIATE;
+
+		nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_sg)
+					* sizeof(struct nx_sg);
+
+		rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead,
+				req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+		if (rc)
+			return rc;
+
+		memcpy(csbcpb_aead->cpb.aes_gca.in_pat,
+				csbcpb_aead->cpb.aes_gca.out_pat,
+				AES_BLOCK_SIZE);
+		NX_CPB_FDM(csbcpb_aead) |= NX_FDM_CONTINUATION;
+
+		atomic_inc(&(nx_ctx->stats->aes_ops));
+		atomic64_add(assoclen, &(nx_ctx->stats->aes_bytes));
+
+		processed += to_process;
+	} while (processed < nbytes);
+
+	memcpy(out, csbcpb_aead->cpb.aes_gca.out_pat, AES_BLOCK_SIZE);
+
+	return rc;
+}
+
+static int gmac(struct aead_request *req, const u8 *iv, unsigned int assoclen)
+{
+	int rc;
+	struct nx_crypto_ctx *nx_ctx =
+		crypto_aead_ctx(crypto_aead_reqtfm(req));
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	struct nx_sg *nx_sg;
+	unsigned int nbytes = assoclen;
+	unsigned int processed = 0, to_process;
+	unsigned int max_sg_len;
+
+	/* Set GMAC mode */
+	csbcpb->cpb.hdr.mode = NX_MODE_AES_GMAC;
+
+	NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION;
+
+	/* page_limit: number of sg entries that fit on one page */
+	max_sg_len = min_t(u64, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+			   nx_ctx->ap->sglen);
+	max_sg_len = min_t(u64, max_sg_len,
+			   nx_ctx->ap->databytelen/NX_PAGE_SIZE);
+
+	/* Copy IV */
+	memcpy(csbcpb->cpb.aes_gcm.iv_or_cnt, iv, AES_BLOCK_SIZE);
+
+	do {
+		/*
+		 * to_process: the data chunk to process in this update.
+		 * This value is bound by sg list limits.
+		 */
+		to_process = min_t(u64, nbytes - processed,
+				   nx_ctx->ap->databytelen);
+		to_process = min_t(u64, to_process,
+				   NX_PAGE_SIZE * (max_sg_len - 1));
+
+		nx_sg = nx_walk_and_build(nx_ctx->in_sg, max_sg_len,
+					  req->src, processed, &to_process);
+
+		if ((to_process + processed) < nbytes)
+			NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+		else
+			NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+		nx_ctx->op.inlen = (nx_ctx->in_sg - nx_sg)
+					* sizeof(struct nx_sg);
+
+		csbcpb->cpb.aes_gcm.bit_length_data = 0;
+		csbcpb->cpb.aes_gcm.bit_length_aad = 8 * nbytes;
+
+		rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+				req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+		if (rc)
+			goto out;
+
+		memcpy(csbcpb->cpb.aes_gcm.in_pat_or_aad,
+			csbcpb->cpb.aes_gcm.out_pat_or_mac, AES_BLOCK_SIZE);
+		memcpy(csbcpb->cpb.aes_gcm.in_s0,
+			csbcpb->cpb.aes_gcm.out_s0, AES_BLOCK_SIZE);
+
+		NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+		atomic_inc(&(nx_ctx->stats->aes_ops));
+		atomic64_add(assoclen, &(nx_ctx->stats->aes_bytes));
+
+		processed += to_process;
+	} while (processed < nbytes);
+
+out:
+	/* Restore GCM mode */
+	csbcpb->cpb.hdr.mode = NX_MODE_AES_GCM;
+	return rc;
+}
+
+static int gcm_empty(struct aead_request *req, const u8 *iv, int enc)
+{
+	int rc;
+	struct nx_crypto_ctx *nx_ctx =
+		crypto_aead_ctx(crypto_aead_reqtfm(req));
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	char out[AES_BLOCK_SIZE];
+	struct nx_sg *in_sg, *out_sg;
+	int len;
+
+	/* For scenarios where the input message is zero length, AES CTR mode
+	 * may be used. Set the source data to be a single block (16B) of all
+	 * zeros, and set the input IV value to be the same as the GMAC IV
+	 * value. - nx_wb 4.8.1.3 */
+
+	/* Change to ECB mode */
+	csbcpb->cpb.hdr.mode = NX_MODE_AES_ECB;
+	memcpy(csbcpb->cpb.aes_ecb.key, csbcpb->cpb.aes_gcm.key,
+			sizeof(csbcpb->cpb.aes_ecb.key));
+	if (enc)
+		NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+	else
+		NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+
+	len = AES_BLOCK_SIZE;
+
+	/* Encrypt the counter/IV */
+	in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) iv,
+				 &len, nx_ctx->ap->sglen);
+
+	if (len != AES_BLOCK_SIZE)
+		return -EINVAL;
+
+	len = sizeof(out);
+	out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *) out, &len,
+				  nx_ctx->ap->sglen);
+
+	if (len != sizeof(out))
+		return -EINVAL;
+
+	nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+	nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+	rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+			   req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+	if (rc)
+		goto out;
+	atomic_inc(&(nx_ctx->stats->aes_ops));
+
+	/* Copy out the auth tag */
+	memcpy(csbcpb->cpb.aes_gcm.out_pat_or_mac, out,
+			crypto_aead_authsize(crypto_aead_reqtfm(req)));
+out:
+	/* Restore XCBC mode */
+	csbcpb->cpb.hdr.mode = NX_MODE_AES_GCM;
+
+	/*
+	 * ECB key uses the same region that GCM AAD and counter, so it's safe
+	 * to just fill it with zeroes.
+	 */
+	memset(csbcpb->cpb.aes_ecb.key, 0, sizeof(csbcpb->cpb.aes_ecb.key));
+
+	return rc;
+}
+
+static int gcm_aes_nx_crypt(struct aead_request *req, int enc,
+			    unsigned int assoclen)
+{
+	struct nx_crypto_ctx *nx_ctx =
+		crypto_aead_ctx(crypto_aead_reqtfm(req));
+	struct nx_gcm_rctx *rctx = aead_request_ctx(req);
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	unsigned int nbytes = req->cryptlen;
+	unsigned int processed = 0, to_process;
+	unsigned long irq_flags;
+	int rc = -EINVAL;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	/* initialize the counter */
+	*(u32 *)&rctx->iv[NX_GCM_CTR_OFFSET] = 1;
+
+	if (nbytes == 0) {
+		if (assoclen == 0)
+			rc = gcm_empty(req, rctx->iv, enc);
+		else
+			rc = gmac(req, rctx->iv, assoclen);
+		if (rc)
+			goto out;
+		else
+			goto mac;
+	}
+
+	/* Process associated data */
+	csbcpb->cpb.aes_gcm.bit_length_aad = assoclen * 8;
+	if (assoclen) {
+		rc = nx_gca(nx_ctx, req, csbcpb->cpb.aes_gcm.in_pat_or_aad,
+			    assoclen);
+		if (rc)
+			goto out;
+	}
+
+	/* Set flags for encryption */
+	NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION;
+	if (enc) {
+		NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+	} else {
+		NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+		nbytes -= crypto_aead_authsize(crypto_aead_reqtfm(req));
+	}
+
+	do {
+		to_process = nbytes - processed;
+
+		csbcpb->cpb.aes_gcm.bit_length_data = nbytes * 8;
+		rc = nx_build_sg_lists(nx_ctx, rctx->iv, req->dst,
+				       req->src, &to_process,
+				       processed + req->assoclen,
+				       csbcpb->cpb.aes_gcm.iv_or_cnt);
+
+		if (rc)
+			goto out;
+
+		if ((to_process + processed) < nbytes)
+			NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+		else
+			NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+
+		rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+				   req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+		if (rc)
+			goto out;
+
+		memcpy(rctx->iv, csbcpb->cpb.aes_gcm.out_cnt, AES_BLOCK_SIZE);
+		memcpy(csbcpb->cpb.aes_gcm.in_pat_or_aad,
+			csbcpb->cpb.aes_gcm.out_pat_or_mac, AES_BLOCK_SIZE);
+		memcpy(csbcpb->cpb.aes_gcm.in_s0,
+			csbcpb->cpb.aes_gcm.out_s0, AES_BLOCK_SIZE);
+
+		NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+		atomic_inc(&(nx_ctx->stats->aes_ops));
+		atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count),
+			     &(nx_ctx->stats->aes_bytes));
+
+		processed += to_process;
+	} while (processed < nbytes);
+
+mac:
+	if (enc) {
+		/* copy out the auth tag */
+		memcpy_to_sglist(
+			req->dst, req->assoclen + nbytes,
+			csbcpb->cpb.aes_gcm.out_pat_or_mac,
+			crypto_aead_authsize(crypto_aead_reqtfm(req)));
+	} else {
+		u8 *itag = nx_ctx->priv.gcm.iauth_tag;
+		u8 *otag = csbcpb->cpb.aes_gcm.out_pat_or_mac;
+
+		memcpy_from_sglist(
+			itag, req->src, req->assoclen + nbytes,
+			crypto_aead_authsize(crypto_aead_reqtfm(req)));
+		rc = crypto_memneq(itag, otag,
+			    crypto_aead_authsize(crypto_aead_reqtfm(req))) ?
+		     -EBADMSG : 0;
+	}
+out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+	return rc;
+}
+
+static int gcm_aes_nx_encrypt(struct aead_request *req)
+{
+	struct nx_gcm_rctx *rctx = aead_request_ctx(req);
+	char *iv = rctx->iv;
+
+	memcpy(iv, req->iv, GCM_AES_IV_SIZE);
+
+	return gcm_aes_nx_crypt(req, 1, req->assoclen);
+}
+
+static int gcm_aes_nx_decrypt(struct aead_request *req)
+{
+	struct nx_gcm_rctx *rctx = aead_request_ctx(req);
+	char *iv = rctx->iv;
+
+	memcpy(iv, req->iv, GCM_AES_IV_SIZE);
+
+	return gcm_aes_nx_crypt(req, 0, req->assoclen);
+}
+
+static int gcm4106_aes_nx_encrypt(struct aead_request *req)
+{
+	struct nx_crypto_ctx *nx_ctx =
+		crypto_aead_ctx(crypto_aead_reqtfm(req));
+	struct nx_gcm_rctx *rctx = aead_request_ctx(req);
+	char *iv = rctx->iv;
+	char *nonce = nx_ctx->priv.gcm.nonce;
+
+	memcpy(iv, nonce, NX_GCM4106_NONCE_LEN);
+	memcpy(iv + NX_GCM4106_NONCE_LEN, req->iv, 8);
+
+	if (req->assoclen < 8)
+		return -EINVAL;
+
+	return gcm_aes_nx_crypt(req, 1, req->assoclen - 8);
+}
+
+static int gcm4106_aes_nx_decrypt(struct aead_request *req)
+{
+	struct nx_crypto_ctx *nx_ctx =
+		crypto_aead_ctx(crypto_aead_reqtfm(req));
+	struct nx_gcm_rctx *rctx = aead_request_ctx(req);
+	char *iv = rctx->iv;
+	char *nonce = nx_ctx->priv.gcm.nonce;
+
+	memcpy(iv, nonce, NX_GCM4106_NONCE_LEN);
+	memcpy(iv + NX_GCM4106_NONCE_LEN, req->iv, 8);
+
+	if (req->assoclen < 8)
+		return -EINVAL;
+
+	return gcm_aes_nx_crypt(req, 0, req->assoclen - 8);
+}
+
+struct aead_alg nx_gcm_aes_alg = {
+	.base = {
+		.cra_name        = "gcm(aes)",
+		.cra_driver_name = "gcm-aes-nx",
+		.cra_priority    = 300,
+		.cra_blocksize   = 1,
+		.cra_ctxsize     = sizeof(struct nx_crypto_ctx),
+		.cra_module      = THIS_MODULE,
+	},
+	.init        = nx_crypto_ctx_aes_gcm_init,
+	.exit        = nx_crypto_ctx_aead_exit,
+	.ivsize      = GCM_AES_IV_SIZE,
+	.maxauthsize = AES_BLOCK_SIZE,
+	.setkey      = gcm_aes_nx_set_key,
+	.encrypt     = gcm_aes_nx_encrypt,
+	.decrypt     = gcm_aes_nx_decrypt,
+};
+
+struct aead_alg nx_gcm4106_aes_alg = {
+	.base = {
+		.cra_name        = "rfc4106(gcm(aes))",
+		.cra_driver_name = "rfc4106-gcm-aes-nx",
+		.cra_priority    = 300,
+		.cra_blocksize   = 1,
+		.cra_ctxsize     = sizeof(struct nx_crypto_ctx),
+		.cra_module      = THIS_MODULE,
+	},
+	.init        = nx_crypto_ctx_aes_gcm_init,
+	.exit        = nx_crypto_ctx_aead_exit,
+	.ivsize      = GCM_RFC4106_IV_SIZE,
+	.maxauthsize = AES_BLOCK_SIZE,
+	.setkey      = gcm4106_aes_nx_set_key,
+	.setauthsize = gcm4106_aes_nx_setauthsize,
+	.encrypt     = gcm4106_aes_nx_encrypt,
+	.decrypt     = gcm4106_aes_nx_decrypt,
+};

linux-7.2.3/drivers/crypto/nx/nx-aes-xcbc.c

@@ -0,0 +1,331 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AES XCBC routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2011-2012 International Business Machines Inc.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/aes.h>
+#include <crypto/internal/hash.h>
+#include <linux/atomic.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+
+struct xcbc_state {
+	u8 state[AES_BLOCK_SIZE];
+};
+
+static int nx_xcbc_set_key(struct crypto_shash *desc,
+			   const u8            *in_key,
+			   unsigned int         key_len)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc);
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+
+	switch (key_len) {
+	case AES_KEYSIZE_128:
+		nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	memcpy(csbcpb->cpb.aes_xcbc.key, in_key, key_len);
+
+	return 0;
+}
+
+/*
+ * Based on RFC 3566, for a zero-length message:
+ *
+ * n = 1
+ * K1 = E(K, 0x01010101010101010101010101010101)
+ * K3 = E(K, 0x03030303030303030303030303030303)
+ * E[0] = 0x00000000000000000000000000000000
+ * M[1] = 0x80000000000000000000000000000000 (0 length message with padding)
+ * E[1] = (K1, M[1] ^ E[0] ^ K3)
+ * Tag = M[1]
+ */
+static int nx_xcbc_empty(struct shash_desc *desc, u8 *out)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm);
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	struct nx_sg *in_sg, *out_sg;
+	u8 keys[2][AES_BLOCK_SIZE];
+	u8 key[32];
+	int rc = 0;
+	int len;
+
+	/* Change to ECB mode */
+	csbcpb->cpb.hdr.mode = NX_MODE_AES_ECB;
+	memcpy(key, csbcpb->cpb.aes_xcbc.key, AES_BLOCK_SIZE);
+	memcpy(csbcpb->cpb.aes_ecb.key, key, AES_BLOCK_SIZE);
+	NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+
+	/* K1 and K3 base patterns */
+	memset(keys[0], 0x01, sizeof(keys[0]));
+	memset(keys[1], 0x03, sizeof(keys[1]));
+
+	len = sizeof(keys);
+	/* Generate K1 and K3 encrypting the patterns */
+	in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) keys, &len,
+				 nx_ctx->ap->sglen);
+
+	if (len != sizeof(keys))
+		return -EINVAL;
+
+	out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *) keys, &len,
+				  nx_ctx->ap->sglen);
+
+	if (len != sizeof(keys))
+		return -EINVAL;
+
+	nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+	nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+	rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0);
+	if (rc)
+		goto out;
+	atomic_inc(&(nx_ctx->stats->aes_ops));
+
+	/* XOr K3 with the padding for a 0 length message */
+	keys[1][0] ^= 0x80;
+
+	len = sizeof(keys[1]);
+
+	/* Encrypt the final result */
+	memcpy(csbcpb->cpb.aes_ecb.key, keys[0], AES_BLOCK_SIZE);
+	in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) keys[1], &len,
+				 nx_ctx->ap->sglen);
+
+	if (len != sizeof(keys[1]))
+		return -EINVAL;
+
+	len = AES_BLOCK_SIZE;
+	out_sg = nx_build_sg_list(nx_ctx->out_sg, out, &len,
+				  nx_ctx->ap->sglen);
+
+	if (len != AES_BLOCK_SIZE)
+		return -EINVAL;
+
+	nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+	nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+	rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0);
+	if (rc)
+		goto out;
+	atomic_inc(&(nx_ctx->stats->aes_ops));
+
+out:
+	/* Restore XCBC mode */
+	csbcpb->cpb.hdr.mode = NX_MODE_AES_XCBC_MAC;
+	memcpy(csbcpb->cpb.aes_xcbc.key, key, AES_BLOCK_SIZE);
+	NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+
+	return rc;
+}
+
+static int nx_crypto_ctx_aes_xcbc_init2(struct crypto_shash *tfm)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm);
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	int err;
+
+	err = nx_crypto_ctx_aes_xcbc_init(tfm);
+	if (err)
+		return err;
+
+	nx_ctx_init(nx_ctx, HCOP_FC_AES);
+
+	NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
+	csbcpb->cpb.hdr.mode = NX_MODE_AES_XCBC_MAC;
+
+	return 0;
+}
+
+static int nx_xcbc_init(struct shash_desc *desc)
+{
+	struct xcbc_state *sctx = shash_desc_ctx(desc);
+
+	memset(sctx, 0, sizeof *sctx);
+
+	return 0;
+}
+
+static int nx_xcbc_update(struct shash_desc *desc,
+			  const u8          *data,
+			  unsigned int       len)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm);
+	struct xcbc_state *sctx = shash_desc_ctx(desc);
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	struct nx_sg *in_sg;
+	struct nx_sg *out_sg;
+	unsigned int max_sg_len;
+	unsigned long irq_flags;
+	u32 to_process, total;
+	int rc = 0;
+	int data_len;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	memcpy(csbcpb->cpb.aes_xcbc.out_cv_mac, sctx->state, AES_BLOCK_SIZE);
+	NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+	NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+	total = len;
+
+	in_sg = nx_ctx->in_sg;
+	max_sg_len = min_t(u64, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+				nx_ctx->ap->sglen);
+	max_sg_len = min_t(u64, max_sg_len,
+				nx_ctx->ap->databytelen/NX_PAGE_SIZE);
+
+	data_len = AES_BLOCK_SIZE;
+	out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *)sctx->state,
+				  &data_len, nx_ctx->ap->sglen);
+
+	if (data_len != AES_BLOCK_SIZE) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+	do {
+		to_process = total & ~(AES_BLOCK_SIZE - 1);
+
+		in_sg = nx_build_sg_list(in_sg,
+					(u8 *) data,
+					&to_process,
+					max_sg_len);
+
+		nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
+					sizeof(struct nx_sg);
+
+		/* we've hit the nx chip previously and we're updating again,
+		 * so copy over the partial digest */
+		memcpy(csbcpb->cpb.aes_xcbc.cv,
+		       csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE);
+
+		if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+			rc = -EINVAL;
+			goto out;
+		}
+
+		rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0);
+		if (rc)
+			goto out;
+
+		atomic_inc(&(nx_ctx->stats->aes_ops));
+
+		total -= to_process;
+		data += to_process;
+		in_sg = nx_ctx->in_sg;
+	} while (total >= AES_BLOCK_SIZE);
+
+	rc = total;
+	memcpy(sctx->state, csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE);
+
+out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+	return rc;
+}
+
+static int nx_xcbc_finup(struct shash_desc *desc, const u8 *src,
+			 unsigned int nbytes, u8 *out)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm);
+	struct xcbc_state *sctx = shash_desc_ctx(desc);
+	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	struct nx_sg *in_sg, *out_sg;
+	unsigned long irq_flags;
+	int rc = 0;
+	int len;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	if (nbytes) {
+		/* non-zero final, so copy over the partial digest */
+		memcpy(csbcpb->cpb.aes_xcbc.cv, sctx->state, AES_BLOCK_SIZE);
+	} else {
+		/*
+		 * we've never seen an update, so this is a 0 byte op. The
+		 * hardware cannot handle a 0 byte op, so just ECB to
+		 * generate the hash.
+		 */
+		rc = nx_xcbc_empty(desc, out);
+		goto out;
+	}
+
+	/* final is represented by continuing the operation and indicating that
+	 * this is not an intermediate operation */
+	NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+	len = nbytes;
+	in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len,
+				 nx_ctx->ap->sglen);
+
+	if (len != nbytes) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	len = AES_BLOCK_SIZE;
+	out_sg = nx_build_sg_list(nx_ctx->out_sg, out, &len,
+				  nx_ctx->ap->sglen);
+
+	if (len != AES_BLOCK_SIZE) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+	nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+	if (!nx_ctx->op.outlen) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0);
+	if (rc)
+		goto out;
+
+	atomic_inc(&(nx_ctx->stats->aes_ops));
+
+	memcpy(out, csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE);
+out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+	return rc;
+}
+
+struct shash_alg nx_shash_aes_xcbc_alg = {
+	.digestsize = AES_BLOCK_SIZE,
+	.init       = nx_xcbc_init,
+	.update     = nx_xcbc_update,
+	.finup      = nx_xcbc_finup,
+	.setkey     = nx_xcbc_set_key,
+	.descsize   = sizeof(struct xcbc_state),
+	.init_tfm   = nx_crypto_ctx_aes_xcbc_init2,
+	.exit_tfm   = nx_crypto_ctx_shash_exit,
+	.base       = {
+		.cra_name        = "xcbc(aes)",
+		.cra_driver_name = "xcbc-aes-nx",
+		.cra_priority    = 300,
+		.cra_flags	 = CRYPTO_AHASH_ALG_BLOCK_ONLY |
+				   CRYPTO_AHASH_ALG_FINAL_NONZERO,
+		.cra_blocksize   = AES_BLOCK_SIZE,
+		.cra_module      = THIS_MODULE,
+		.cra_ctxsize     = sizeof(struct nx_crypto_ctx),
+	}
+};

linux-7.2.3/drivers/crypto/nx/nx-sha256.c

@@ -0,0 +1,260 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * SHA-256 routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2011-2012 International Business Machines Inc.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/internal/hash.h>
+#include <crypto/sha2.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <linux/unaligned.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+struct sha256_state_be {
+	__be32 state[SHA256_DIGEST_SIZE / 4];
+	u64 count;
+};
+
+static int nx_crypto_ctx_sha256_init(struct crypto_shash *tfm)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm);
+	int err;
+
+	err = nx_crypto_ctx_sha_init(tfm);
+	if (err)
+		return err;
+
+	nx_ctx_init(nx_ctx, HCOP_FC_SHA);
+
+	nx_ctx->ap = &nx_ctx->props[NX_PROPS_SHA256];
+
+	NX_CPB_SET_DIGEST_SIZE(nx_ctx->csbcpb, NX_DS_SHA256);
+
+	return 0;
+}
+
+static int nx_sha256_init(struct shash_desc *desc)
+{
+	struct sha256_state_be *sctx = shash_desc_ctx(desc);
+
+	sctx->state[0] = __cpu_to_be32(SHA256_H0);
+	sctx->state[1] = __cpu_to_be32(SHA256_H1);
+	sctx->state[2] = __cpu_to_be32(SHA256_H2);
+	sctx->state[3] = __cpu_to_be32(SHA256_H3);
+	sctx->state[4] = __cpu_to_be32(SHA256_H4);
+	sctx->state[5] = __cpu_to_be32(SHA256_H5);
+	sctx->state[6] = __cpu_to_be32(SHA256_H6);
+	sctx->state[7] = __cpu_to_be32(SHA256_H7);
+	sctx->count = 0;
+
+	return 0;
+}
+
+static int nx_sha256_update(struct shash_desc *desc, const u8 *data,
+			    unsigned int len)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm);
+	struct sha256_state_be *sctx = shash_desc_ctx(desc);
+	struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
+	u64 to_process, leftover, total = len;
+	struct nx_sg *out_sg;
+	unsigned long irq_flags;
+	int rc = 0;
+	int data_len;
+	u32 max_sg_len;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	memcpy(csbcpb->cpb.sha256.message_digest, sctx->state, SHA256_DIGEST_SIZE);
+	NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+	NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+	max_sg_len = min_t(u64, nx_ctx->ap->sglen,
+			nx_driver.of.max_sg_len/sizeof(struct nx_sg));
+	max_sg_len = min_t(u64, max_sg_len,
+			nx_ctx->ap->databytelen/NX_PAGE_SIZE);
+
+	data_len = SHA256_DIGEST_SIZE;
+	out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *)sctx->state,
+				  &data_len, max_sg_len);
+	nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+	if (data_len != SHA256_DIGEST_SIZE) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	do {
+		struct nx_sg *in_sg = nx_ctx->in_sg;
+
+		to_process = total & ~(SHA256_BLOCK_SIZE - 1);
+
+		data_len = to_process;
+		in_sg = nx_build_sg_list(in_sg, (u8 *) data,
+					 &data_len, max_sg_len);
+
+		nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+
+		to_process = data_len;
+		leftover = total - to_process;
+
+		/*
+		 * we've hit the nx chip previously and we're updating
+		 * again, so copy over the partial digest.
+		 */
+		memcpy(csbcpb->cpb.sha256.input_partial_digest,
+			       csbcpb->cpb.sha256.message_digest,
+			       SHA256_DIGEST_SIZE);
+
+		if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+			rc = -EINVAL;
+			goto out;
+		}
+
+		rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0);
+		if (rc)
+			goto out;
+
+		atomic_inc(&(nx_ctx->stats->sha256_ops));
+
+		total -= to_process;
+		data += to_process;
+		sctx->count += to_process;
+	} while (leftover >= SHA256_BLOCK_SIZE);
+
+	rc = leftover;
+	memcpy(sctx->state, csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE);
+out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+	return rc;
+}
+
+static int nx_sha256_finup(struct shash_desc *desc, const u8 *src,
+			   unsigned int nbytes, u8 *out)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm);
+	struct sha256_state_be *sctx = shash_desc_ctx(desc);
+	struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
+	struct nx_sg *in_sg, *out_sg;
+	unsigned long irq_flags;
+	u32 max_sg_len;
+	int rc = 0;
+	int len;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	max_sg_len = min_t(u64, nx_ctx->ap->sglen,
+			nx_driver.of.max_sg_len/sizeof(struct nx_sg));
+	max_sg_len = min_t(u64, max_sg_len,
+			nx_ctx->ap->databytelen/NX_PAGE_SIZE);
+
+	/* final is represented by continuing the operation and indicating that
+	 * this is not an intermediate operation
+	 * copy over the partial digest */
+	memcpy(csbcpb->cpb.sha256.input_partial_digest, sctx->state, SHA256_DIGEST_SIZE);
+	NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+	NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+	sctx->count += nbytes;
+	csbcpb->cpb.sha256.message_bit_length = (u64) (sctx->count * 8);
+
+	len = nbytes;
+	in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, max_sg_len);
+
+	if (len != nbytes) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	len = SHA256_DIGEST_SIZE;
+	out_sg = nx_build_sg_list(nx_ctx->out_sg, out, &len, max_sg_len);
+
+	if (len != SHA256_DIGEST_SIZE) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+	nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+	if (!nx_ctx->op.outlen) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0);
+	if (rc)
+		goto out;
+
+	atomic_inc(&(nx_ctx->stats->sha256_ops));
+
+	atomic64_add(sctx->count, &(nx_ctx->stats->sha256_bytes));
+	memcpy(out, csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE);
+out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+	return rc;
+}
+
+static int nx_sha256_export(struct shash_desc *desc, void *out)
+{
+	struct sha256_state_be *sctx = shash_desc_ctx(desc);
+	union {
+		u8 *u8;
+		u32 *u32;
+		u64 *u64;
+	} p = { .u8 = out };
+	int i;
+
+	for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(*p.u32); i++)
+		put_unaligned(be32_to_cpu(sctx->state[i]), p.u32++);
+
+	put_unaligned(sctx->count, p.u64++);
+	return 0;
+}
+
+static int nx_sha256_import(struct shash_desc *desc, const void *in)
+{
+	struct sha256_state_be *sctx = shash_desc_ctx(desc);
+	union {
+		const u8 *u8;
+		const u32 *u32;
+		const u64 *u64;
+	} p = { .u8 = in };
+	int i;
+
+	for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(*p.u32); i++)
+		sctx->state[i] = cpu_to_be32(get_unaligned(p.u32++));
+
+	sctx->count = get_unaligned(p.u64++);
+	return 0;
+}
+
+struct shash_alg nx_shash_sha256_alg = {
+	.digestsize = SHA256_DIGEST_SIZE,
+	.init       = nx_sha256_init,
+	.update     = nx_sha256_update,
+	.finup      = nx_sha256_finup,
+	.export     = nx_sha256_export,
+	.import     = nx_sha256_import,
+	.init_tfm   = nx_crypto_ctx_sha256_init,
+	.exit_tfm   = nx_crypto_ctx_shash_exit,
+	.descsize   = sizeof(struct sha256_state_be),
+	.statesize  = sizeof(struct sha256_state_be),
+	.base       = {
+		.cra_name        = "sha256",
+		.cra_driver_name = "sha256-nx",
+		.cra_priority    = 300,
+		.cra_flags	 = CRYPTO_AHASH_ALG_BLOCK_ONLY,
+		.cra_blocksize   = SHA256_BLOCK_SIZE,
+		.cra_module      = THIS_MODULE,
+		.cra_ctxsize     = sizeof(struct nx_crypto_ctx),
+	}
+};

linux-7.2.3/drivers/crypto/nx/nx-sha512.c

@@ -0,0 +1,265 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * SHA-512 routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2011-2012 International Business Machines Inc.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/internal/hash.h>
+#include <crypto/sha2.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <linux/unaligned.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+struct sha512_state_be {
+	__be64 state[SHA512_DIGEST_SIZE / 8];
+	u64 count[2];
+};
+
+static int nx_crypto_ctx_sha512_init(struct crypto_shash *tfm)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm);
+	int err;
+
+	err = nx_crypto_ctx_sha_init(tfm);
+	if (err)
+		return err;
+
+	nx_ctx_init(nx_ctx, HCOP_FC_SHA);
+
+	nx_ctx->ap = &nx_ctx->props[NX_PROPS_SHA512];
+
+	NX_CPB_SET_DIGEST_SIZE(nx_ctx->csbcpb, NX_DS_SHA512);
+
+	return 0;
+}
+
+static int nx_sha512_init(struct shash_desc *desc)
+{
+	struct sha512_state_be *sctx = shash_desc_ctx(desc);
+
+	sctx->state[0] = __cpu_to_be64(SHA512_H0);
+	sctx->state[1] = __cpu_to_be64(SHA512_H1);
+	sctx->state[2] = __cpu_to_be64(SHA512_H2);
+	sctx->state[3] = __cpu_to_be64(SHA512_H3);
+	sctx->state[4] = __cpu_to_be64(SHA512_H4);
+	sctx->state[5] = __cpu_to_be64(SHA512_H5);
+	sctx->state[6] = __cpu_to_be64(SHA512_H6);
+	sctx->state[7] = __cpu_to_be64(SHA512_H7);
+	sctx->count[0] = 0;
+	sctx->count[1] = 0;
+
+	return 0;
+}
+
+static int nx_sha512_update(struct shash_desc *desc, const u8 *data,
+			    unsigned int len)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm);
+	struct sha512_state_be *sctx = shash_desc_ctx(desc);
+	struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
+	u64 to_process, leftover, total = len;
+	struct nx_sg *out_sg;
+	unsigned long irq_flags;
+	int rc = 0;
+	int data_len;
+	u32 max_sg_len;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	memcpy(csbcpb->cpb.sha512.message_digest, sctx->state, SHA512_DIGEST_SIZE);
+	NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+	NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+	max_sg_len = min_t(u64, nx_ctx->ap->sglen,
+			nx_driver.of.max_sg_len/sizeof(struct nx_sg));
+	max_sg_len = min_t(u64, max_sg_len,
+			nx_ctx->ap->databytelen/NX_PAGE_SIZE);
+
+	data_len = SHA512_DIGEST_SIZE;
+	out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *)sctx->state,
+				  &data_len, max_sg_len);
+	nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+	if (data_len != SHA512_DIGEST_SIZE) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	do {
+		struct nx_sg *in_sg = nx_ctx->in_sg;
+
+		to_process = total & ~(SHA512_BLOCK_SIZE - 1);
+
+		data_len = to_process;
+		in_sg = nx_build_sg_list(in_sg, (u8 *) data,
+					 &data_len, max_sg_len);
+
+		nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+
+		to_process = data_len;
+		leftover = total - to_process;
+
+		/*
+		 * we've hit the nx chip previously and we're updating
+		 * again, so copy over the partial digest.
+		 */
+		memcpy(csbcpb->cpb.sha512.input_partial_digest,
+			       csbcpb->cpb.sha512.message_digest,
+			       SHA512_DIGEST_SIZE);
+
+		if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+			rc = -EINVAL;
+			goto out;
+		}
+
+		rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0);
+		if (rc)
+			goto out;
+
+		atomic_inc(&(nx_ctx->stats->sha512_ops));
+
+		total -= to_process;
+		data += to_process;
+		sctx->count[0] += to_process;
+		if (sctx->count[0] < to_process)
+			sctx->count[1]++;
+	} while (leftover >= SHA512_BLOCK_SIZE);
+
+	rc = leftover;
+	memcpy(sctx->state, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE);
+out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+	return rc;
+}
+
+static int nx_sha512_finup(struct shash_desc *desc, const u8 *src,
+			   unsigned int nbytes, u8 *out)
+{
+	struct sha512_state_be *sctx = shash_desc_ctx(desc);
+	struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm);
+	struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
+	struct nx_sg *in_sg, *out_sg;
+	u32 max_sg_len;
+	unsigned long irq_flags;
+	u64 count0, count1;
+	int rc = 0;
+	int len;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	max_sg_len = min_t(u64, nx_ctx->ap->sglen,
+			nx_driver.of.max_sg_len/sizeof(struct nx_sg));
+	max_sg_len = min_t(u64, max_sg_len,
+			nx_ctx->ap->databytelen/NX_PAGE_SIZE);
+
+	/* final is represented by continuing the operation and indicating that
+	 * this is not an intermediate operation
+	 * copy over the partial digest */
+	memcpy(csbcpb->cpb.sha512.input_partial_digest, sctx->state, SHA512_DIGEST_SIZE);
+	NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+	NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+	count0 = sctx->count[0] + nbytes;
+	count1 = sctx->count[1];
+
+	csbcpb->cpb.sha512.message_bit_length_lo = count0 << 3;
+	csbcpb->cpb.sha512.message_bit_length_hi = (count1 << 3) |
+						   (count0 >> 61);
+
+	len = nbytes;
+	in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, max_sg_len);
+
+	if (len != nbytes) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	len = SHA512_DIGEST_SIZE;
+	out_sg = nx_build_sg_list(nx_ctx->out_sg, out, &len,
+				 max_sg_len);
+
+	nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+	nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+	if (!nx_ctx->op.outlen) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0);
+	if (rc)
+		goto out;
+
+	atomic_inc(&(nx_ctx->stats->sha512_ops));
+	atomic64_add(count0, &(nx_ctx->stats->sha512_bytes));
+
+	memcpy(out, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE);
+out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+	return rc;
+}
+
+static int nx_sha512_export(struct shash_desc *desc, void *out)
+{
+	struct sha512_state_be *sctx = shash_desc_ctx(desc);
+	union {
+		u8 *u8;
+		u64 *u64;
+	} p = { .u8 = out };
+	int i;
+
+	for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(*p.u64); i++)
+		put_unaligned(be64_to_cpu(sctx->state[i]), p.u64++);
+
+	put_unaligned(sctx->count[0], p.u64++);
+	put_unaligned(sctx->count[1], p.u64++);
+	return 0;
+}
+
+static int nx_sha512_import(struct shash_desc *desc, const void *in)
+{
+	struct sha512_state_be *sctx = shash_desc_ctx(desc);
+	union {
+		const u8 *u8;
+		const u64 *u64;
+	} p = { .u8 = in };
+	int i;
+
+	for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(*p.u64); i++)
+		sctx->state[i] = cpu_to_be64(get_unaligned(p.u64++));
+
+	sctx->count[0] = get_unaligned(p.u64++);
+	sctx->count[1] = get_unaligned(p.u64++);
+	return 0;
+}
+
+struct shash_alg nx_shash_sha512_alg = {
+	.digestsize = SHA512_DIGEST_SIZE,
+	.init       = nx_sha512_init,
+	.update     = nx_sha512_update,
+	.finup      = nx_sha512_finup,
+	.export     = nx_sha512_export,
+	.import     = nx_sha512_import,
+	.init_tfm   = nx_crypto_ctx_sha512_init,
+	.exit_tfm   = nx_crypto_ctx_shash_exit,
+	.descsize   = sizeof(struct sha512_state_be),
+	.statesize  = sizeof(struct sha512_state_be),
+	.base       = {
+		.cra_name        = "sha512",
+		.cra_driver_name = "sha512-nx",
+		.cra_priority    = 300,
+		.cra_flags	 = CRYPTO_AHASH_ALG_BLOCK_ONLY,
+		.cra_blocksize   = SHA512_BLOCK_SIZE,
+		.cra_module      = THIS_MODULE,
+		.cra_ctxsize     = sizeof(struct nx_crypto_ctx),
+	}
+};

linux-7.2.3/drivers/crypto/nx/nx.c

@@ -0,0 +1,830 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2011-2012 International Business Machines Inc.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/aes.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/hash.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/sha2.h>
+#include <crypto/scatterwalk.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+#include <linux/mm.h>
+#include <linux/scatterlist.h>
+#include <linux/device.h>
+#include <linux/of.h>
+#include <asm/hvcall.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+
+/**
+ * nx_hcall_sync - make an H_COP_OP hcall for the passed in op structure
+ *
+ * @nx_ctx: the crypto context handle
+ * @op: PFO operation struct to pass in
+ * @may_sleep: flag indicating the request can sleep
+ *
+ * Make the hcall, retrying while the hardware is busy. If we cannot yield
+ * the thread, limit the number of retries to 10 here.
+ */
+int nx_hcall_sync(struct nx_crypto_ctx *nx_ctx,
+		  struct vio_pfo_op    *op,
+		  u32                   may_sleep)
+{
+	int rc, retries = 10;
+	struct vio_dev *viodev = nx_driver.viodev;
+
+	atomic_inc(&(nx_ctx->stats->sync_ops));
+
+	do {
+		rc = vio_h_cop_sync(viodev, op);
+	} while (rc == -EBUSY && !may_sleep && retries--);
+
+	if (rc) {
+		dev_dbg(&viodev->dev, "vio_h_cop_sync failed: rc: %d "
+			"hcall rc: %ld\n", rc, op->hcall_err);
+		atomic_inc(&(nx_ctx->stats->errors));
+		atomic_set(&(nx_ctx->stats->last_error), op->hcall_err);
+		atomic_set(&(nx_ctx->stats->last_error_pid), current->pid);
+	}
+
+	return rc;
+}
+
+/**
+ * nx_build_sg_list - build an NX scatter list describing a single  buffer
+ *
+ * @sg_head: pointer to the first scatter list element to build
+ * @start_addr: pointer to the linear buffer
+ * @len: length of the data at @start_addr
+ * @sgmax: the largest number of scatter list elements we're allowed to create
+ *
+ * This function will start writing nx_sg elements at @sg_head and keep
+ * writing them until all of the data from @start_addr is described or
+ * until sgmax elements have been written. Scatter list elements will be
+ * created such that none of the elements describes a buffer that crosses a 4K
+ * boundary.
+ */
+struct nx_sg *nx_build_sg_list(struct nx_sg *sg_head,
+			       u8           *start_addr,
+			       unsigned int *len,
+			       u32           sgmax)
+{
+	unsigned int sg_len = 0;
+	struct nx_sg *sg;
+	u64 sg_addr = (u64)start_addr;
+	u64 end_addr;
+
+	/* determine the start and end for this address range - slightly
+	 * different if this is in VMALLOC_REGION */
+	if (is_vmalloc_addr(start_addr))
+		sg_addr = page_to_phys(vmalloc_to_page(start_addr))
+			  + offset_in_page(sg_addr);
+	else
+		sg_addr = __pa(sg_addr);
+
+	end_addr = sg_addr + *len;
+
+	/* each iteration will write one struct nx_sg element and add the
+	 * length of data described by that element to sg_len. Once @len bytes
+	 * have been described (or @sgmax elements have been written), the
+	 * loop ends. min_t is used to ensure @end_addr falls on the same page
+	 * as sg_addr, if not, we need to create another nx_sg element for the
+	 * data on the next page.
+	 *
+	 * Also when using vmalloc'ed data, every time that a system page
+	 * boundary is crossed the physical address needs to be re-calculated.
+	 */
+	for (sg = sg_head; sg_len < *len; sg++) {
+		u64 next_page;
+
+		sg->addr = sg_addr;
+		sg_addr = min_t(u64, NX_PAGE_NUM(sg_addr + NX_PAGE_SIZE),
+				end_addr);
+
+		next_page = (sg->addr & PAGE_MASK) + PAGE_SIZE;
+		sg->len = min_t(u64, sg_addr, next_page) - sg->addr;
+		sg_len += sg->len;
+
+		if (sg_addr >= next_page &&
+				is_vmalloc_addr(start_addr + sg_len)) {
+			sg_addr = page_to_phys(vmalloc_to_page(
+						start_addr + sg_len));
+			end_addr = sg_addr + *len - sg_len;
+		}
+
+		if ((sg - sg_head) == sgmax) {
+			sg++;
+			break;
+		}
+	}
+	*len = sg_len;
+
+	/* return the moved sg_head pointer */
+	return sg;
+}
+
+/**
+ * nx_walk_and_build - walk a linux scatterlist and build an nx scatterlist
+ *
+ * @nx_dst: pointer to the first nx_sg element to write
+ * @sglen: max number of nx_sg entries we're allowed to write
+ * @sg_src: pointer to the source linux scatterlist to walk
+ * @start: number of bytes to fast-forward past at the beginning of @sg_src
+ * @src_len: number of bytes to walk in @sg_src
+ */
+struct nx_sg *nx_walk_and_build(struct nx_sg       *nx_dst,
+				unsigned int        sglen,
+				struct scatterlist *sg_src,
+				unsigned int        start,
+				unsigned int       *src_len)
+{
+	struct scatter_walk walk;
+	struct nx_sg *nx_sg = nx_dst;
+	unsigned int n, len = *src_len;
+
+	/* we need to fast forward through @start bytes first */
+	scatterwalk_start_at_pos(&walk, sg_src, start);
+
+	while (len && (nx_sg - nx_dst) < sglen) {
+		n = scatterwalk_next(&walk, len);
+
+		nx_sg = nx_build_sg_list(nx_sg, walk.addr, &n, sglen - (nx_sg - nx_dst));
+
+		scatterwalk_done_src(&walk, n);
+		len -= n;
+	}
+	/* update to_process */
+	*src_len -= len;
+
+	/* return the moved destination pointer */
+	return nx_sg;
+}
+
+/**
+ * trim_sg_list - ensures the bound in sg list.
+ * @sg: sg list head
+ * @end: sg lisg end
+ * @delta:  is the amount we need to crop in order to bound the list.
+ * @nbytes: length of data in the scatterlists or data length - whichever
+ *          is greater.
+ */
+static long int trim_sg_list(struct nx_sg *sg,
+			     struct nx_sg *end,
+			     unsigned int delta,
+			     unsigned int *nbytes)
+{
+	long int oplen;
+	long int data_back;
+	unsigned int is_delta = delta;
+
+	while (delta && end > sg) {
+		struct nx_sg *last = end - 1;
+
+		if (last->len > delta) {
+			last->len -= delta;
+			delta = 0;
+		} else {
+			end--;
+			delta -= last->len;
+		}
+	}
+
+	/* There are cases where we need to crop list in order to make it
+	 * a block size multiple, but we also need to align data. In order to
+	 * that we need to calculate how much we need to put back to be
+	 * processed
+	 */
+	oplen = (sg - end) * sizeof(struct nx_sg);
+	if (is_delta) {
+		data_back = (abs(oplen) / AES_BLOCK_SIZE) *  sg->len;
+		data_back = *nbytes - (data_back & ~(AES_BLOCK_SIZE - 1));
+		*nbytes -= data_back;
+	}
+
+	return oplen;
+}
+
+/**
+ * nx_build_sg_lists - walk the input scatterlists and build arrays of NX
+ *                     scatterlists based on them.
+ *
+ * @nx_ctx: NX crypto context for the lists we're building
+ * @iv: iv data, if the algorithm requires it
+ * @dst: destination scatterlist
+ * @src: source scatterlist
+ * @nbytes: length of data described in the scatterlists
+ * @offset: number of bytes to fast-forward past at the beginning of
+ *          scatterlists.
+ * @oiv: destination for the iv data, if the algorithm requires it
+ *
+ * This is common code shared by all the AES algorithms. It uses the crypto
+ * scatterlist walk routines to traverse input and output scatterlists, building
+ * corresponding NX scatterlists
+ */
+int nx_build_sg_lists(struct nx_crypto_ctx  *nx_ctx,
+		      const u8              *iv,
+		      struct scatterlist    *dst,
+		      struct scatterlist    *src,
+		      unsigned int          *nbytes,
+		      unsigned int           offset,
+		      u8                    *oiv)
+{
+	unsigned int delta = 0;
+	unsigned int total = *nbytes;
+	struct nx_sg *nx_insg = nx_ctx->in_sg;
+	struct nx_sg *nx_outsg = nx_ctx->out_sg;
+	unsigned int max_sg_len;
+
+	max_sg_len = min_t(u64, nx_ctx->ap->sglen,
+			nx_driver.of.max_sg_len/sizeof(struct nx_sg));
+	max_sg_len = min_t(u64, max_sg_len,
+			nx_ctx->ap->databytelen/NX_PAGE_SIZE);
+
+	if (oiv)
+		memcpy(oiv, iv, AES_BLOCK_SIZE);
+
+	*nbytes = min_t(u64, *nbytes, nx_ctx->ap->databytelen);
+
+	nx_outsg = nx_walk_and_build(nx_outsg, max_sg_len, dst,
+					offset, nbytes);
+	nx_insg = nx_walk_and_build(nx_insg, max_sg_len, src,
+					offset, nbytes);
+
+	if (*nbytes < total)
+		delta = *nbytes - (*nbytes & ~(AES_BLOCK_SIZE - 1));
+
+	/* these lengths should be negative, which will indicate to phyp that
+	 * the input and output parameters are scatterlists, not linear
+	 * buffers */
+	nx_ctx->op.inlen = trim_sg_list(nx_ctx->in_sg, nx_insg, delta, nbytes);
+	nx_ctx->op.outlen = trim_sg_list(nx_ctx->out_sg, nx_outsg, delta, nbytes);
+
+	return 0;
+}
+
+/**
+ * nx_ctx_init - initialize an nx_ctx's vio_pfo_op struct
+ *
+ * @nx_ctx: the nx context to initialize
+ * @function: the function code for the op
+ */
+void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function)
+{
+	spin_lock_init(&nx_ctx->lock);
+	memset(nx_ctx->kmem, 0, nx_ctx->kmem_len);
+	nx_ctx->csbcpb->csb.valid |= NX_CSB_VALID_BIT;
+
+	nx_ctx->op.flags = function;
+	nx_ctx->op.csbcpb = __pa(nx_ctx->csbcpb);
+	nx_ctx->op.in = __pa(nx_ctx->in_sg);
+	nx_ctx->op.out = __pa(nx_ctx->out_sg);
+
+	if (nx_ctx->csbcpb_aead) {
+		nx_ctx->csbcpb_aead->csb.valid |= NX_CSB_VALID_BIT;
+
+		nx_ctx->op_aead.flags = function;
+		nx_ctx->op_aead.csbcpb = __pa(nx_ctx->csbcpb_aead);
+		nx_ctx->op_aead.in = __pa(nx_ctx->in_sg);
+		nx_ctx->op_aead.out = __pa(nx_ctx->out_sg);
+	}
+}
+
+static void nx_of_update_status(struct device   *dev,
+			       struct property *p,
+			       struct nx_of    *props)
+{
+	if (!strncmp(p->value, "okay", p->length)) {
+		props->status = NX_WAITING;
+		props->flags |= NX_OF_FLAG_STATUS_SET;
+	} else {
+		dev_info(dev, "%s: status '%s' is not 'okay'\n", __func__,
+			 (char *)p->value);
+	}
+}
+
+static void nx_of_update_sglen(struct device   *dev,
+			       struct property *p,
+			       struct nx_of    *props)
+{
+	if (p->length != sizeof(props->max_sg_len)) {
+		dev_err(dev, "%s: unexpected format for "
+			"ibm,max-sg-len property\n", __func__);
+		dev_dbg(dev, "%s: ibm,max-sg-len is %d bytes "
+			"long, expected %zd bytes\n", __func__,
+			p->length, sizeof(props->max_sg_len));
+		return;
+	}
+
+	props->max_sg_len = *(u32 *)p->value;
+	props->flags |= NX_OF_FLAG_MAXSGLEN_SET;
+}
+
+static void nx_of_update_msc(struct device   *dev,
+			     struct property *p,
+			     struct nx_of    *props)
+{
+	struct msc_triplet *trip;
+	struct max_sync_cop *msc;
+	unsigned int bytes_so_far, i, lenp;
+
+	msc = (struct max_sync_cop *)p->value;
+	lenp = p->length;
+
+	/* You can't tell if the data read in for this property is sane by its
+	 * size alone. This is because there are sizes embedded in the data
+	 * structure. The best we can do is check lengths as we parse and bail
+	 * as soon as a length error is detected. */
+	bytes_so_far = 0;
+
+	while ((bytes_so_far + sizeof(struct max_sync_cop)) <= lenp) {
+		bytes_so_far += sizeof(struct max_sync_cop);
+
+		trip = msc->trip;
+
+		for (i = 0;
+		     ((bytes_so_far + sizeof(struct msc_triplet)) <= lenp) &&
+		     i < msc->triplets;
+		     i++) {
+			if (msc->fc >= NX_MAX_FC || msc->mode >= NX_MAX_MODE) {
+				dev_err(dev, "unknown function code/mode "
+					"combo: %d/%d (ignored)\n", msc->fc,
+					msc->mode);
+				goto next_loop;
+			}
+
+			if (!trip->sglen || trip->databytelen < NX_PAGE_SIZE) {
+				dev_warn(dev, "bogus sglen/databytelen: "
+					 "%u/%u (ignored)\n", trip->sglen,
+					 trip->databytelen);
+				goto next_loop;
+			}
+
+			switch (trip->keybitlen) {
+			case 128:
+			case 160:
+				props->ap[msc->fc][msc->mode][0].databytelen =
+					trip->databytelen;
+				props->ap[msc->fc][msc->mode][0].sglen =
+					trip->sglen;
+				break;
+			case 192:
+				props->ap[msc->fc][msc->mode][1].databytelen =
+					trip->databytelen;
+				props->ap[msc->fc][msc->mode][1].sglen =
+					trip->sglen;
+				break;
+			case 256:
+				if (msc->fc == NX_FC_AES) {
+					props->ap[msc->fc][msc->mode][2].
+						databytelen = trip->databytelen;
+					props->ap[msc->fc][msc->mode][2].sglen =
+						trip->sglen;
+				} else if (msc->fc == NX_FC_AES_HMAC ||
+					   msc->fc == NX_FC_SHA) {
+					props->ap[msc->fc][msc->mode][1].
+						databytelen = trip->databytelen;
+					props->ap[msc->fc][msc->mode][1].sglen =
+						trip->sglen;
+				} else {
+					dev_warn(dev, "unknown function "
+						"code/key bit len combo"
+						": (%u/256)\n", msc->fc);
+				}
+				break;
+			case 512:
+				props->ap[msc->fc][msc->mode][2].databytelen =
+					trip->databytelen;
+				props->ap[msc->fc][msc->mode][2].sglen =
+					trip->sglen;
+				break;
+			default:
+				dev_warn(dev, "unknown function code/key bit "
+					 "len combo: (%u/%u)\n", msc->fc,
+					 trip->keybitlen);
+				break;
+			}
+next_loop:
+			bytes_so_far += sizeof(struct msc_triplet);
+			trip++;
+		}
+
+		msc = (struct max_sync_cop *)trip;
+	}
+
+	props->flags |= NX_OF_FLAG_MAXSYNCCOP_SET;
+}
+
+/**
+ * nx_of_init - read openFirmware values from the device tree
+ *
+ * @dev: device handle
+ * @props: pointer to struct to hold the properties values
+ *
+ * Called once at driver probe time, this function will read out the
+ * openFirmware properties we use at runtime. If all the OF properties are
+ * acceptable, when we exit this function props->flags will indicate that
+ * we're ready to register our crypto algorithms.
+ */
+static void nx_of_init(struct device *dev, struct nx_of *props)
+{
+	struct device_node *base_node = dev->of_node;
+	struct property *p;
+
+	p = of_find_property(base_node, "status", NULL);
+	if (!p)
+		dev_info(dev, "%s: property 'status' not found\n", __func__);
+	else
+		nx_of_update_status(dev, p, props);
+
+	p = of_find_property(base_node, "ibm,max-sg-len", NULL);
+	if (!p)
+		dev_info(dev, "%s: property 'ibm,max-sg-len' not found\n",
+			 __func__);
+	else
+		nx_of_update_sglen(dev, p, props);
+
+	p = of_find_property(base_node, "ibm,max-sync-cop", NULL);
+	if (!p)
+		dev_info(dev, "%s: property 'ibm,max-sync-cop' not found\n",
+			 __func__);
+	else
+		nx_of_update_msc(dev, p, props);
+}
+
+static bool nx_check_prop(struct device *dev, u32 fc, u32 mode, int slot)
+{
+	struct alg_props *props = &nx_driver.of.ap[fc][mode][slot];
+
+	if (!props->sglen || props->databytelen < NX_PAGE_SIZE) {
+		if (dev)
+			dev_warn(dev, "bogus sglen/databytelen for %u/%u/%u: "
+				 "%u/%u (ignored)\n", fc, mode, slot,
+				 props->sglen, props->databytelen);
+		return false;
+	}
+
+	return true;
+}
+
+static bool nx_check_props(struct device *dev, u32 fc, u32 mode)
+{
+	int i;
+
+	for (i = 0; i < 3; i++)
+		if (!nx_check_prop(dev, fc, mode, i))
+			return false;
+
+	return true;
+}
+
+static int nx_register_skcipher(struct skcipher_alg *alg, u32 fc, u32 mode)
+{
+	return nx_check_props(&nx_driver.viodev->dev, fc, mode) ?
+	       crypto_register_skcipher(alg) : 0;
+}
+
+static int nx_register_aead(struct aead_alg *alg, u32 fc, u32 mode)
+{
+	return nx_check_props(&nx_driver.viodev->dev, fc, mode) ?
+	       crypto_register_aead(alg) : 0;
+}
+
+static int nx_register_shash(struct shash_alg *alg, u32 fc, u32 mode, int slot)
+{
+	return (slot >= 0 ? nx_check_prop(&nx_driver.viodev->dev,
+					  fc, mode, slot) :
+			    nx_check_props(&nx_driver.viodev->dev, fc, mode)) ?
+	       crypto_register_shash(alg) : 0;
+}
+
+static void nx_unregister_skcipher(struct skcipher_alg *alg, u32 fc, u32 mode)
+{
+	if (nx_check_props(NULL, fc, mode))
+		crypto_unregister_skcipher(alg);
+}
+
+static void nx_unregister_aead(struct aead_alg *alg, u32 fc, u32 mode)
+{
+	if (nx_check_props(NULL, fc, mode))
+		crypto_unregister_aead(alg);
+}
+
+static void nx_unregister_shash(struct shash_alg *alg, u32 fc, u32 mode,
+				int slot)
+{
+	if (slot >= 0 ? nx_check_prop(NULL, fc, mode, slot) :
+			nx_check_props(NULL, fc, mode))
+		crypto_unregister_shash(alg);
+}
+
+/**
+ * nx_register_algs - register algorithms with the crypto API
+ *
+ * Called from nx_probe()
+ *
+ * If all OF properties are in an acceptable state, the driver flags will
+ * indicate that we're ready and we'll create our debugfs files and register
+ * out crypto algorithms.
+ */
+static int nx_register_algs(void)
+{
+	int rc = -1;
+
+	if (nx_driver.of.flags != NX_OF_FLAG_MASK_READY)
+		goto out;
+
+	memset(&nx_driver.stats, 0, sizeof(struct nx_stats));
+
+	NX_DEBUGFS_INIT(&nx_driver);
+
+	nx_driver.of.status = NX_OKAY;
+
+	rc = nx_register_skcipher(&nx_ecb_aes_alg, NX_FC_AES, NX_MODE_AES_ECB);
+	if (rc)
+		goto out;
+
+	rc = nx_register_skcipher(&nx_cbc_aes_alg, NX_FC_AES, NX_MODE_AES_CBC);
+	if (rc)
+		goto out_unreg_ecb;
+
+	rc = nx_register_skcipher(&nx_ctr3686_aes_alg, NX_FC_AES,
+				  NX_MODE_AES_CTR);
+	if (rc)
+		goto out_unreg_cbc;
+
+	rc = nx_register_aead(&nx_gcm_aes_alg, NX_FC_AES, NX_MODE_AES_GCM);
+	if (rc)
+		goto out_unreg_ctr3686;
+
+	rc = nx_register_aead(&nx_gcm4106_aes_alg, NX_FC_AES, NX_MODE_AES_GCM);
+	if (rc)
+		goto out_unreg_gcm;
+
+	rc = nx_register_aead(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
+	if (rc)
+		goto out_unreg_gcm4106;
+
+	rc = nx_register_aead(&nx_ccm4309_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
+	if (rc)
+		goto out_unreg_ccm;
+
+	rc = nx_register_shash(&nx_shash_sha256_alg, NX_FC_SHA, NX_MODE_SHA,
+			       NX_PROPS_SHA256);
+	if (rc)
+		goto out_unreg_ccm4309;
+
+	rc = nx_register_shash(&nx_shash_sha512_alg, NX_FC_SHA, NX_MODE_SHA,
+			       NX_PROPS_SHA512);
+	if (rc)
+		goto out_unreg_s256;
+
+	rc = nx_register_shash(&nx_shash_aes_xcbc_alg,
+			       NX_FC_AES, NX_MODE_AES_XCBC_MAC, -1);
+	if (rc)
+		goto out_unreg_s512;
+
+	goto out;
+
+out_unreg_s512:
+	nx_unregister_shash(&nx_shash_sha512_alg, NX_FC_SHA, NX_MODE_SHA,
+			    NX_PROPS_SHA512);
+out_unreg_s256:
+	nx_unregister_shash(&nx_shash_sha256_alg, NX_FC_SHA, NX_MODE_SHA,
+			    NX_PROPS_SHA256);
+out_unreg_ccm4309:
+	nx_unregister_aead(&nx_ccm4309_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
+out_unreg_ccm:
+	nx_unregister_aead(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
+out_unreg_gcm4106:
+	nx_unregister_aead(&nx_gcm4106_aes_alg, NX_FC_AES, NX_MODE_AES_GCM);
+out_unreg_gcm:
+	nx_unregister_aead(&nx_gcm_aes_alg, NX_FC_AES, NX_MODE_AES_GCM);
+out_unreg_ctr3686:
+	nx_unregister_skcipher(&nx_ctr3686_aes_alg, NX_FC_AES, NX_MODE_AES_CTR);
+out_unreg_cbc:
+	nx_unregister_skcipher(&nx_cbc_aes_alg, NX_FC_AES, NX_MODE_AES_CBC);
+out_unreg_ecb:
+	nx_unregister_skcipher(&nx_ecb_aes_alg, NX_FC_AES, NX_MODE_AES_ECB);
+out:
+	return rc;
+}
+
+/**
+ * nx_crypto_ctx_init - create and initialize a crypto api context
+ *
+ * @nx_ctx: the crypto api context
+ * @fc: function code for the context
+ * @mode: the function code specific mode for this context
+ */
+static int nx_crypto_ctx_init(struct nx_crypto_ctx *nx_ctx, u32 fc, u32 mode)
+{
+	if (nx_driver.of.status != NX_OKAY) {
+		pr_err("Attempt to initialize NX crypto context while device "
+		       "is not available!\n");
+		return -ENODEV;
+	}
+
+	/* we need an extra page for csbcpb_aead for these modes */
+	if (mode == NX_MODE_AES_GCM || mode == NX_MODE_AES_CCM)
+		nx_ctx->kmem_len = (5 * NX_PAGE_SIZE) +
+				   sizeof(struct nx_csbcpb);
+	else
+		nx_ctx->kmem_len = (4 * NX_PAGE_SIZE) +
+				   sizeof(struct nx_csbcpb);
+
+	nx_ctx->kmem = kmalloc(nx_ctx->kmem_len, GFP_KERNEL);
+	if (!nx_ctx->kmem)
+		return -ENOMEM;
+
+	/* the csbcpb and scatterlists must be 4K aligned pages */
+	nx_ctx->csbcpb = (struct nx_csbcpb *)(round_up((u64)nx_ctx->kmem,
+						       (u64)NX_PAGE_SIZE));
+	nx_ctx->in_sg = (struct nx_sg *)((u8 *)nx_ctx->csbcpb + NX_PAGE_SIZE);
+	nx_ctx->out_sg = (struct nx_sg *)((u8 *)nx_ctx->in_sg + NX_PAGE_SIZE);
+
+	if (mode == NX_MODE_AES_GCM || mode == NX_MODE_AES_CCM)
+		nx_ctx->csbcpb_aead =
+			(struct nx_csbcpb *)((u8 *)nx_ctx->out_sg +
+					     NX_PAGE_SIZE);
+
+	/* give each context a pointer to global stats and their OF
+	 * properties */
+	nx_ctx->stats = &nx_driver.stats;
+	memcpy(nx_ctx->props, nx_driver.of.ap[fc][mode],
+	       sizeof(struct alg_props) * 3);
+
+	return 0;
+}
+
+/* entry points from the crypto tfm initializers */
+int nx_crypto_ctx_aes_ccm_init(struct crypto_aead *tfm)
+{
+	crypto_aead_set_reqsize(tfm, sizeof(struct nx_ccm_rctx));
+	return nx_crypto_ctx_init(crypto_aead_ctx(tfm), NX_FC_AES,
+				  NX_MODE_AES_CCM);
+}
+
+int nx_crypto_ctx_aes_gcm_init(struct crypto_aead *tfm)
+{
+	crypto_aead_set_reqsize(tfm, sizeof(struct nx_gcm_rctx));
+	return nx_crypto_ctx_init(crypto_aead_ctx(tfm), NX_FC_AES,
+				  NX_MODE_AES_GCM);
+}
+
+int nx_crypto_ctx_aes_ctr_init(struct crypto_skcipher *tfm)
+{
+	return nx_crypto_ctx_init(crypto_skcipher_ctx(tfm), NX_FC_AES,
+				  NX_MODE_AES_CTR);
+}
+
+int nx_crypto_ctx_aes_cbc_init(struct crypto_skcipher *tfm)
+{
+	return nx_crypto_ctx_init(crypto_skcipher_ctx(tfm), NX_FC_AES,
+				  NX_MODE_AES_CBC);
+}
+
+int nx_crypto_ctx_aes_ecb_init(struct crypto_skcipher *tfm)
+{
+	return nx_crypto_ctx_init(crypto_skcipher_ctx(tfm), NX_FC_AES,
+				  NX_MODE_AES_ECB);
+}
+
+int nx_crypto_ctx_sha_init(struct crypto_shash *tfm)
+{
+	return nx_crypto_ctx_init(crypto_shash_ctx(tfm), NX_FC_SHA, NX_MODE_SHA);
+}
+
+int nx_crypto_ctx_aes_xcbc_init(struct crypto_shash *tfm)
+{
+	return nx_crypto_ctx_init(crypto_shash_ctx(tfm), NX_FC_AES,
+				  NX_MODE_AES_XCBC_MAC);
+}
+
+/**
+ * nx_crypto_ctx_exit - destroy a crypto api context
+ *
+ * @nx_ctx: the crypto api context
+ *
+ * As crypto API contexts are destroyed, this exit hook is called to free the
+ * memory associated with it.
+ */
+void nx_crypto_ctx_exit(struct nx_crypto_ctx *nx_ctx)
+{
+	kfree_sensitive(nx_ctx->kmem);
+	nx_ctx->csbcpb = NULL;
+	nx_ctx->csbcpb_aead = NULL;
+	nx_ctx->in_sg = NULL;
+	nx_ctx->out_sg = NULL;
+}
+
+void nx_crypto_ctx_skcipher_exit(struct crypto_skcipher *tfm)
+{
+	nx_crypto_ctx_exit(crypto_skcipher_ctx(tfm));
+}
+
+void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm)
+{
+	struct nx_crypto_ctx *nx_ctx = crypto_aead_ctx(tfm);
+
+	kfree_sensitive(nx_ctx->kmem);
+}
+
+void nx_crypto_ctx_shash_exit(struct crypto_shash *tfm)
+{
+	nx_crypto_ctx_exit(crypto_shash_ctx(tfm));
+}
+
+static int nx_probe(struct vio_dev *viodev, const struct vio_device_id *id)
+{
+	dev_dbg(&viodev->dev, "driver probed: %s resource id: 0x%x\n",
+		viodev->name, viodev->resource_id);
+
+	if (nx_driver.viodev) {
+		dev_err(&viodev->dev, "%s: Attempt to register more than one "
+			"instance of the hardware\n", __func__);
+		return -EINVAL;
+	}
+
+	nx_driver.viodev = viodev;
+
+	nx_of_init(&viodev->dev, &nx_driver.of);
+
+	return nx_register_algs();
+}
+
+static void nx_remove(struct vio_dev *viodev)
+{
+	dev_dbg(&viodev->dev, "entering nx_remove for UA 0x%x\n",
+		viodev->unit_address);
+
+	if (nx_driver.of.status == NX_OKAY) {
+		NX_DEBUGFS_FINI(&nx_driver);
+
+		nx_unregister_shash(&nx_shash_aes_xcbc_alg,
+				    NX_FC_AES, NX_MODE_AES_XCBC_MAC, -1);
+		nx_unregister_shash(&nx_shash_sha512_alg,
+				    NX_FC_SHA, NX_MODE_SHA, NX_PROPS_SHA256);
+		nx_unregister_shash(&nx_shash_sha256_alg,
+				    NX_FC_SHA, NX_MODE_SHA, NX_PROPS_SHA512);
+		nx_unregister_aead(&nx_ccm4309_aes_alg,
+				   NX_FC_AES, NX_MODE_AES_CCM);
+		nx_unregister_aead(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
+		nx_unregister_aead(&nx_gcm4106_aes_alg,
+				   NX_FC_AES, NX_MODE_AES_GCM);
+		nx_unregister_aead(&nx_gcm_aes_alg,
+				   NX_FC_AES, NX_MODE_AES_GCM);
+		nx_unregister_skcipher(&nx_ctr3686_aes_alg,
+				       NX_FC_AES, NX_MODE_AES_CTR);
+		nx_unregister_skcipher(&nx_cbc_aes_alg, NX_FC_AES,
+				       NX_MODE_AES_CBC);
+		nx_unregister_skcipher(&nx_ecb_aes_alg, NX_FC_AES,
+				       NX_MODE_AES_ECB);
+	}
+}
+
+
+/* module wide initialization/cleanup */
+static int __init nx_init(void)
+{
+	return vio_register_driver(&nx_driver.viodriver);
+}
+
+static void __exit nx_fini(void)
+{
+	vio_unregister_driver(&nx_driver.viodriver);
+}
+
+static const struct vio_device_id nx_crypto_driver_ids[] = {
+	{ "ibm,sym-encryption-v1", "ibm,sym-encryption" },
+	{ "", "" }
+};
+MODULE_DEVICE_TABLE(vio, nx_crypto_driver_ids);
+
+/* driver state structure */
+struct nx_crypto_driver nx_driver = {
+	.viodriver = {
+		.id_table = nx_crypto_driver_ids,
+		.probe = nx_probe,
+		.remove = nx_remove,
+		.name  = NX_NAME,
+	},
+};
+
+module_init(nx_init);
+module_exit(nx_fini);
+
+MODULE_AUTHOR("Kent Yoder <yoder1@us.ibm.com>");
+MODULE_DESCRIPTION(NX_STRING);
+MODULE_LICENSE("GPL");
+MODULE_VERSION(NX_VERSION);

linux-7.2.3/drivers/crypto/nx/nx.h

@@ -0,0 +1,197 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __NX_H__
+#define __NX_H__
+
+#include <asm/vio.h>
+#include <crypto/ctr.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/hash.h>
+#include <crypto/internal/skcipher.h>
+
+#define NX_NAME		"nx-crypto"
+#define NX_STRING	"IBM Power7+ Nest Accelerator Crypto Driver"
+#define NX_VERSION	"1.0"
+
+/* a scatterlist in the format PHYP is expecting */
+struct nx_sg {
+	u64 addr;
+	u32 rsvd;
+	u32 len;
+} __attribute((packed));
+
+#define NX_PAGE_SIZE		(4096)
+#define NX_MAX_SG_ENTRIES	(NX_PAGE_SIZE/(sizeof(struct nx_sg)))
+
+enum nx_status {
+	NX_DISABLED,
+	NX_WAITING,
+	NX_OKAY
+};
+
+/* msc_triplet and max_sync_cop are used only to assist in parsing the
+ * openFirmware property */
+struct msc_triplet {
+	u32 keybitlen;
+	u32 databytelen;
+	u32 sglen;
+} __packed;
+
+struct max_sync_cop {
+	u32 fc;
+	u32 mode;
+	u32 triplets;
+	struct msc_triplet trip[];
+} __packed;
+
+struct alg_props {
+	u32 databytelen;
+	u32 sglen;
+};
+
+#define NX_OF_FLAG_MAXSGLEN_SET		(1)
+#define NX_OF_FLAG_STATUS_SET		(2)
+#define NX_OF_FLAG_MAXSYNCCOP_SET	(4)
+#define NX_OF_FLAG_MASK_READY		(NX_OF_FLAG_MAXSGLEN_SET | \
+					 NX_OF_FLAG_STATUS_SET |   \
+					 NX_OF_FLAG_MAXSYNCCOP_SET)
+struct nx_of {
+	u32 flags;
+	u32 max_sg_len;
+	enum nx_status status;
+	struct alg_props ap[NX_MAX_FC][NX_MAX_MODE][3];
+};
+
+struct nx_stats {
+	atomic_t aes_ops;
+	atomic64_t aes_bytes;
+	atomic_t sha256_ops;
+	atomic64_t sha256_bytes;
+	atomic_t sha512_ops;
+	atomic64_t sha512_bytes;
+
+	atomic_t sync_ops;
+
+	atomic_t errors;
+	atomic_t last_error;
+	atomic_t last_error_pid;
+};
+
+struct nx_crypto_driver {
+	struct nx_stats    stats;
+	struct nx_of       of;
+	struct vio_dev    *viodev;
+	struct vio_driver  viodriver;
+	struct dentry     *dfs_root;
+};
+
+#define NX_GCM4106_NONCE_LEN		(4)
+#define NX_GCM_CTR_OFFSET		(12)
+struct nx_gcm_rctx {
+	u8 iv[16];
+};
+
+struct nx_gcm_priv {
+	u8 iauth_tag[16];
+	u8 nonce[NX_GCM4106_NONCE_LEN];
+};
+
+#define NX_CCM_AES_KEY_LEN		(16)
+#define NX_CCM4309_AES_KEY_LEN		(19)
+#define NX_CCM4309_NONCE_LEN		(3)
+struct nx_ccm_rctx {
+	u8 iv[16];
+};
+
+struct nx_ccm_priv {
+	u8 b0[16];
+	u8 iauth_tag[16];
+	u8 oauth_tag[16];
+	u8 nonce[NX_CCM4309_NONCE_LEN];
+};
+
+struct nx_xcbc_priv {
+	u8 key[16];
+};
+
+struct nx_ctr_priv {
+	u8 nonce[CTR_RFC3686_NONCE_SIZE];
+};
+
+struct nx_crypto_ctx {
+	spinlock_t lock;	  /* synchronize access to the context */
+	void *kmem;		  /* unaligned, kmalloc'd buffer */
+	size_t kmem_len;	  /* length of kmem */
+	struct nx_csbcpb *csbcpb; /* aligned page given to phyp @ hcall time */
+	struct vio_pfo_op op;     /* operation struct with hcall parameters */
+	struct nx_csbcpb *csbcpb_aead; /* secondary csbcpb used by AEAD algs */
+	struct vio_pfo_op op_aead;/* operation struct for csbcpb_aead */
+
+	struct nx_sg *in_sg;      /* aligned pointer into kmem to an sg list */
+	struct nx_sg *out_sg;     /* aligned pointer into kmem to an sg list */
+
+	struct alg_props *ap;	  /* pointer into props based on our key size */
+	struct alg_props props[3];/* openFirmware properties for requests */
+	struct nx_stats *stats;   /* pointer into an nx_crypto_driver for stats
+				     reporting */
+
+	union {
+		struct nx_gcm_priv gcm;
+		struct nx_ccm_priv ccm;
+		struct nx_xcbc_priv xcbc;
+		struct nx_ctr_priv ctr;
+	} priv;
+};
+
+struct scatterlist;
+
+/* prototypes */
+int nx_crypto_ctx_aes_ccm_init(struct crypto_aead *tfm);
+int nx_crypto_ctx_aes_gcm_init(struct crypto_aead *tfm);
+int nx_crypto_ctx_aes_xcbc_init(struct crypto_shash *tfm);
+int nx_crypto_ctx_aes_ctr_init(struct crypto_skcipher *tfm);
+int nx_crypto_ctx_aes_cbc_init(struct crypto_skcipher *tfm);
+int nx_crypto_ctx_aes_ecb_init(struct crypto_skcipher *tfm);
+int nx_crypto_ctx_sha_init(struct crypto_shash *tfm);
+void nx_crypto_ctx_exit(struct nx_crypto_ctx *nx_ctx);
+void nx_crypto_ctx_skcipher_exit(struct crypto_skcipher *tfm);
+void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm);
+void nx_crypto_ctx_shash_exit(struct crypto_shash *tfm);
+void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function);
+int nx_hcall_sync(struct nx_crypto_ctx *ctx, struct vio_pfo_op *op,
+		  u32 may_sleep);
+struct nx_sg *nx_build_sg_list(struct nx_sg *, u8 *, unsigned int *, u32);
+int nx_build_sg_lists(struct nx_crypto_ctx *nx_ctx, const u8 *iv,
+		      struct scatterlist *dst, struct scatterlist *src,
+		      unsigned int *nbytes, unsigned int offset, u8 *oiv);
+struct nx_sg *nx_walk_and_build(struct nx_sg *, unsigned int,
+				struct scatterlist *, unsigned int,
+				unsigned int *);
+
+#ifdef CONFIG_DEBUG_FS
+#define NX_DEBUGFS_INIT(drv)	nx_debugfs_init(drv)
+#define NX_DEBUGFS_FINI(drv)	nx_debugfs_fini(drv)
+
+void nx_debugfs_init(struct nx_crypto_driver *);
+void nx_debugfs_fini(struct nx_crypto_driver *);
+#else
+#define NX_DEBUGFS_INIT(drv)	do {} while (0)
+#define NX_DEBUGFS_FINI(drv)	do {} while (0)
+#endif
+
+#define NX_PAGE_NUM(x)		((u64)(x) & 0xfffffffffffff000ULL)
+
+extern struct skcipher_alg nx_cbc_aes_alg;
+extern struct skcipher_alg nx_ecb_aes_alg;
+extern struct aead_alg nx_gcm_aes_alg;
+extern struct aead_alg nx_gcm4106_aes_alg;
+extern struct skcipher_alg nx_ctr3686_aes_alg;
+extern struct aead_alg nx_ccm_aes_alg;
+extern struct aead_alg nx_ccm4309_aes_alg;
+extern struct shash_alg nx_shash_aes_xcbc_alg;
+extern struct shash_alg nx_shash_sha512_alg;
+extern struct shash_alg nx_shash_sha256_alg;
+
+extern struct nx_crypto_driver nx_driver;
+
+#endif

linux-7.2.3/drivers/crypto/nx/nx_csbcpb.h

@@ -0,0 +1,206 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __NX_CSBCPB_H__
+#define __NX_CSBCPB_H__
+
+struct cop_symcpb_aes_ecb {
+	u8 key[32];
+	u8 __rsvd[80];
+} __packed;
+
+struct cop_symcpb_aes_cbc {
+	u8 iv[16];
+	u8 key[32];
+	u8 cv[16];
+	u32 spbc;
+	u8 __rsvd[44];
+} __packed;
+
+struct cop_symcpb_aes_gca {
+	u8 in_pat[16];
+	u8 key[32];
+	u8 out_pat[16];
+	u32 spbc;
+	u8 __rsvd[44];
+} __packed;
+
+struct cop_symcpb_aes_gcm {
+	u8 in_pat_or_aad[16];
+	u8 iv_or_cnt[16];
+	u64 bit_length_aad;
+	u64 bit_length_data;
+	u8 in_s0[16];
+	u8 key[32];
+	u8 __rsvd1[16];
+	u8 out_pat_or_mac[16];
+	u8 out_s0[16];
+	u8 out_cnt[16];
+	u32 spbc;
+	u8 __rsvd2[12];
+} __packed;
+
+struct cop_symcpb_aes_ctr {
+	u8 iv[16];
+	u8 key[32];
+	u8 cv[16];
+	u32 spbc;
+	u8 __rsvd2[44];
+} __packed;
+
+struct cop_symcpb_aes_cca {
+	u8 b0[16];
+	u8 b1[16];
+	u8 key[16];
+	u8 out_pat_or_b0[16];
+	u32 spbc;
+	u8 __rsvd[44];
+} __packed;
+
+struct cop_symcpb_aes_ccm {
+	u8 in_pat_or_b0[16];
+	u8 iv_or_ctr[16];
+	u8 in_s0[16];
+	u8 key[16];
+	u8 __rsvd1[48];
+	u8 out_pat_or_mac[16];
+	u8 out_s0[16];
+	u8 out_ctr[16];
+	u32 spbc;
+	u8 __rsvd2[12];
+} __packed;
+
+struct cop_symcpb_aes_xcbc {
+	u8 cv[16];
+	u8 key[16];
+	u8 __rsvd1[16];
+	u8 out_cv_mac[16];
+	u32 spbc;
+	u8 __rsvd2[44];
+} __packed;
+
+struct cop_symcpb_sha256 {
+	u64 message_bit_length;
+	u64 __rsvd1;
+	u8 input_partial_digest[32];
+	u8 message_digest[32];
+	u32 spbc;
+	u8 __rsvd2[44];
+} __packed;
+
+struct cop_symcpb_sha512 {
+	u64 message_bit_length_hi;
+	u64 message_bit_length_lo;
+	u8 input_partial_digest[64];
+	u8 __rsvd1[32];
+	u8 message_digest[64];
+	u32 spbc;
+	u8 __rsvd2[76];
+} __packed;
+
+#define NX_FDM_INTERMEDIATE		0x01
+#define NX_FDM_CONTINUATION		0x02
+#define NX_FDM_ENDE_ENCRYPT		0x80
+
+#define NX_CPB_FDM(c)			((c)->cpb.hdr.fdm)
+#define NX_CPB_KS_DS(c)			((c)->cpb.hdr.ks_ds)
+
+#define NX_CPB_KEY_SIZE(c)		(NX_CPB_KS_DS(c) >> 4)
+#define NX_CPB_SET_KEY_SIZE(c, x)	NX_CPB_KS_DS(c) |= ((x) << 4)
+#define NX_CPB_SET_DIGEST_SIZE(c, x)	NX_CPB_KS_DS(c) |= (x)
+
+struct cop_symcpb_header {
+	u8 mode;
+	u8 fdm;
+	u8 ks_ds;
+	u8 pad_byte;
+	u8 __rsvd[12];
+} __packed;
+
+struct cop_parameter_block {
+	struct cop_symcpb_header hdr;
+	union {
+		struct cop_symcpb_aes_ecb  aes_ecb;
+		struct cop_symcpb_aes_cbc  aes_cbc;
+		struct cop_symcpb_aes_gca  aes_gca;
+		struct cop_symcpb_aes_gcm  aes_gcm;
+		struct cop_symcpb_aes_cca  aes_cca;
+		struct cop_symcpb_aes_ccm  aes_ccm;
+		struct cop_symcpb_aes_ctr  aes_ctr;
+		struct cop_symcpb_aes_xcbc aes_xcbc;
+		struct cop_symcpb_sha256   sha256;
+		struct cop_symcpb_sha512   sha512;
+	};
+} __packed;
+
+#define NX_CSB_VALID_BIT	0x80
+
+/* co-processor status block */
+struct cop_status_block {
+	u8 valid;
+	u8 crb_seq_number;
+	u8 completion_code;
+	u8 completion_extension;
+	__be32 processed_byte_count;
+	__be64 address;
+} __packed;
+
+/* Nest accelerator workbook section 4.4 */
+struct nx_csbcpb {
+	unsigned char __rsvd[112];
+	struct cop_status_block csb;
+	struct cop_parameter_block cpb;
+} __packed;
+
+/* nx_csbcpb related definitions */
+#define NX_MODE_AES_ECB			0
+#define NX_MODE_AES_CBC			1
+#define NX_MODE_AES_GMAC		2
+#define NX_MODE_AES_GCA			3
+#define NX_MODE_AES_GCM			4
+#define NX_MODE_AES_CCA			5
+#define NX_MODE_AES_CCM			6
+#define NX_MODE_AES_CTR			7
+#define NX_MODE_AES_XCBC_MAC		20
+#define NX_MODE_SHA			0
+#define NX_MODE_SHA_HMAC		1
+#define NX_MODE_AES_CBC_HMAC_ETA	8
+#define NX_MODE_AES_CBC_HMAC_ATE	9
+#define NX_MODE_AES_CBC_HMAC_EAA	10
+#define NX_MODE_AES_CTR_HMAC_ETA	12
+#define NX_MODE_AES_CTR_HMAC_ATE	13
+#define NX_MODE_AES_CTR_HMAC_EAA	14
+
+#define NX_FDM_CI_FULL		0
+#define NX_FDM_CI_FIRST		1
+#define NX_FDM_CI_LAST		2
+#define NX_FDM_CI_MIDDLE	3
+
+#define NX_FDM_PR_NONE		0
+#define NX_FDM_PR_PAD		1
+
+#define NX_KS_AES_128		1
+#define NX_KS_AES_192		2
+#define NX_KS_AES_256		3
+
+#define NX_DS_SHA256		2
+#define NX_DS_SHA512		3
+
+#define NX_FC_AES		0
+#define NX_FC_SHA		2
+#define NX_FC_AES_HMAC		6
+
+#define NX_MAX_FC		(NX_FC_AES_HMAC + 1)
+#define NX_MAX_MODE		(NX_MODE_AES_XCBC_MAC + 1)
+
+#define HCOP_FC_AES          NX_FC_AES
+#define HCOP_FC_SHA          NX_FC_SHA
+#define HCOP_FC_AES_HMAC     NX_FC_AES_HMAC
+
+/* indices into the array of algorithm properties */
+#define NX_PROPS_AES_128		0
+#define NX_PROPS_AES_192		1
+#define NX_PROPS_AES_256		2
+#define NX_PROPS_SHA256			1
+#define NX_PROPS_SHA512			2
+
+#endif

linux-7.2.3/drivers/crypto/nx/nx_debugfs.c

@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * debugfs routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2011-2012 International Business Machines Inc.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <linux/device.h>
+#include <linux/kobject.h>
+#include <linux/string.h>
+#include <linux/debugfs.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/crypto.h>
+#include <crypto/hash.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+#ifdef CONFIG_DEBUG_FS
+
+/*
+ * debugfs
+ *
+ * For documentation on these attributes, please see:
+ *
+ * Documentation/ABI/testing/debugfs-pfo-nx-crypto
+ */
+
+void nx_debugfs_init(struct nx_crypto_driver *drv)
+{
+	struct dentry *root;
+
+	root = debugfs_create_dir(NX_NAME, NULL);
+	drv->dfs_root = root;
+
+	debugfs_create_u32("aes_ops", S_IRUSR | S_IRGRP | S_IROTH,
+			   root, &drv->stats.aes_ops.counter);
+	debugfs_create_u32("sha256_ops", S_IRUSR | S_IRGRP | S_IROTH,
+			   root, &drv->stats.sha256_ops.counter);
+	debugfs_create_u32("sha512_ops", S_IRUSR | S_IRGRP | S_IROTH,
+			   root, &drv->stats.sha512_ops.counter);
+	debugfs_create_u64("aes_bytes", S_IRUSR | S_IRGRP | S_IROTH,
+			   root, &drv->stats.aes_bytes.counter);
+	debugfs_create_u64("sha256_bytes", S_IRUSR | S_IRGRP | S_IROTH,
+			   root, &drv->stats.sha256_bytes.counter);
+	debugfs_create_u64("sha512_bytes", S_IRUSR | S_IRGRP | S_IROTH,
+			   root, &drv->stats.sha512_bytes.counter);
+	debugfs_create_u32("errors", S_IRUSR | S_IRGRP | S_IROTH,
+			   root, &drv->stats.errors.counter);
+	debugfs_create_u32("last_error", S_IRUSR | S_IRGRP | S_IROTH,
+			   root, &drv->stats.last_error.counter);
+	debugfs_create_u32("last_error_pid", S_IRUSR | S_IRGRP | S_IROTH,
+			   root, &drv->stats.last_error_pid.counter);
+}
+
+void
+nx_debugfs_fini(struct nx_crypto_driver *drv)
+{
+	debugfs_remove_recursive(drv->dfs_root);
+}
+
+#endif