From 87a880672b6fc8800949ba36a59c73e59237597f Mon Sep 17 00:00:00 2001 From: Karsten Koenig Date: Mon, 17 Feb 2025 15:30:28 +0100 Subject: [PATCH] nrf_security: cracen: Improve DCACHE handling Calling plain invalidate isn't always safe, as there might be pending writes in the cache line that just get thrown away. Just switch to full cache line evasion, so writeback + invalidate just to be safe. Also iterate over the out descriptors to make sure all the buffers are cache handled on completing of the cracen operations. Signed-off-by: Karsten Koenig --- .../src/drivers/cracen/sxsymcrypt/src/aead.c | 3 +- .../drivers/cracen/sxsymcrypt/src/blkcipher.c | 15 +++++----- .../src/drivers/cracen/sxsymcrypt/src/cmdma.c | 21 ++++++++++++-- .../src/drivers/cracen/sxsymcrypt/src/cmdma.h | 29 +++++++++++-------- .../src/drivers/cracen/sxsymcrypt/src/hash.c | 5 ++-- .../src/drivers/cracen/sxsymcrypt/src/mac.c | 3 +- 6 files changed, 50 insertions(+), 26 deletions(-) diff --git a/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/aead.c b/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/aead.c index 7c4d46804058..2e6eb4d4c9e5 100644 --- a/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/aead.c +++ b/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/aead.c @@ -474,7 +474,8 @@ int sx_aead_status(struct sxaead *c) } #if CONFIG_DCACHE - sys_cache_data_invd_range((void *)&c->extramem, sizeof(c->extramem)); + sx_cmdma_outdescs_flush_and_invd_dcache(&c->dma); + sys_cache_data_flush_and_invd_range((void *)&c->extramem, sizeof(c->extramem)); #endif if (!(c->dma.dmamem.cfg & c->cfg->ctxsave) && c->expectedtag != NULL) { diff --git a/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/blkcipher.c b/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/blkcipher.c index ae4a23df8c7f..44d6ddc57881 100644 --- a/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/blkcipher.c +++ b/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/blkcipher.c @@ -86,7 +86,6 @@ void sx_blkcipher_free(struct sxblkcipher *c) sx_cmdma_release_hw(&c->dma); } - static int sx_blkcipher_hw_reserve(struct sxblkcipher *c) { int err = SX_OK; @@ -149,8 +148,7 @@ static int sx_blkcipher_create_aesxts(struct sxblkcipher *c, const struct sxkeyr mode = CMDMA_BLKCIPHER_MODE_SET(BLKCIPHER_MODEID_XTS); keyszfld = 0; - sx_cmdma_newcmd(&c->dma, c->descs, - KEYREF_AES_HWKEY_CONF(key1->cfg) | mode | keyszfld, + sx_cmdma_newcmd(&c->dma, c->descs, KEYREF_AES_HWKEY_CONF(key1->cfg) | mode | keyszfld, c->cfg->dmatags->cfg); if (KEYREF_IS_USR(key1)) { ADD_CFGDESC(c->dma, key1->key, key1->sz, c->cfg->dmatags->key); @@ -240,8 +238,7 @@ int sx_blkcipher_create_aesctr_dec(struct sxblkcipher *c, const struct sxkeyref return sx_blkcipher_create_aes_ba411(c, key, iv, &ba411ctrcfg, ba411ctrcfg.decr); } -int sx_blkcipher_create_aesecb_enc(struct sxblkcipher *c, const struct sxkeyref *key - ) +int sx_blkcipher_create_aesecb_enc(struct sxblkcipher *c, const struct sxkeyref *key) { return sx_blkcipher_create_aes_ba411(c, key, NULL, &ba411ecbcfg, CM_CFG_ENCRYPT); } @@ -381,7 +378,9 @@ int sx_blkcipher_status(struct sxblkcipher *c) } #if CONFIG_DCACHE - sys_cache_data_invd_range((void *)&c->extramem, sizeof(c->extramem)); + sx_cmdma_outdescs_flush_and_invd_dcache(&c->dma); + /* extramem is often a target for out descriptors, this might not be needed anymore */ + sys_cache_data_flush_and_invd_range((void *)&c->extramem, sizeof(c->extramem)); #endif sx_blkcipher_free(c); @@ -435,6 +434,8 @@ int sx_blkcipher_ecb_simple(uint8_t *key, size_t key_size, uint8_t *input, size_ #if CONFIG_DCACHE sys_cache_data_flush_range(in_descs, sizeof(in_descs)); sys_cache_data_flush_range(&out_desc, sizeof(out_desc)); + sys_cache_data_flush_range(&cmd, sizeof(cmd)); + sys_cache_data_flush_range(key, key_size); sys_cache_data_flush_range(input, input_size); sys_cache_data_flush_range(output, output_size); #endif @@ -451,7 +452,7 @@ int sx_blkcipher_ecb_simple(uint8_t *key, size_t key_size, uint8_t *input, size_ sx_cmdma_release_hw(NULL); #if CONFIG_DCACHE - sys_cache_data_invd_range(output, output_size); + sys_cache_data_flush_and_invd_range(output, output_size); #endif return r; diff --git a/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/cmdma.c b/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/cmdma.c index 2da5cb752711..4c941c76ace0 100644 --- a/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/cmdma.c +++ b/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/cmdma.c @@ -28,8 +28,9 @@ static void sx_cmdma_finalize_descs(struct sxdesc *start, struct sxdesc *end) for (d = start; d < end; d++) { #ifdef DMA_FIFO_ADDR - if (d->addr == (char *)DMA_FIFO_ADDR) + if (d->addr == (char *)DMA_FIFO_ADDR) { d->sz |= DMA_CONST_ADDR; + } #endif d->next = d + 1; } @@ -37,8 +38,9 @@ static void sx_cmdma_finalize_descs(struct sxdesc *start, struct sxdesc *end) end->dmatag |= DMATAG_LAST; end->sz |= DMA_REALIGN; #ifdef DMA_FIFO_ADDR - if (end->addr == (char *)DMA_FIFO_ADDR) + if (end->addr == (char *)DMA_FIFO_ADDR) { end->sz |= DMA_CONST_ADDR; + } #endif } @@ -52,14 +54,16 @@ void sx_cmdma_start(struct sx_dmactl *dma, size_t privsz, struct sxdesc *indescs #ifdef CONFIG_DCACHE m = (struct sxdesc *)(dma->mapped + sizeof(struct sx_dmaslot)); for (; m != DMA_LAST_DESCRIPTOR; m = m->next) { + sys_cache_data_flush_and_invd_range(m, sizeof(*m)); sys_cache_data_flush_and_invd_range(m->addr, m->sz & DMA_SZ_MASK); } m = (struct sxdesc *)(dma->mapped + offsetof(struct sx_dmaslot, outdescs)); for (; m != DMA_LAST_DESCRIPTOR; m = m->next) { + sys_cache_data_flush_and_invd_range(m, sizeof(*m)); sys_cache_data_flush_and_invd_range(m->addr, m->sz & DMA_SZ_MASK); } - sys_cache_data_flush_range((void *)&dma->dmamem, sizeof(dma->dmamem) + privsz); + sys_cache_data_flush_and_invd_range((void *)&dma->dmamem, sizeof(dma->dmamem) + privsz); #endif m = (struct sxdesc *)(dma->mapped + sizeof(struct sx_dmaslot)); @@ -70,6 +74,17 @@ void sx_cmdma_start(struct sx_dmactl *dma, size_t privsz, struct sxdesc *indescs sx_wrreg(REG_START, REG_START_ALL); } +#ifdef CONFIG_DCACHE +void sx_cmdma_outdescs_flush_and_invd_dcache(const struct sx_dmactl *dma) +{ + struct sxdesc *m = (struct sxdesc *)(dma->mapped + offsetof(struct sx_dmaslot, outdescs)); + + for (; m != DMA_LAST_DESCRIPTOR; m = m->next) { + sys_cache_data_flush_and_invd_range(m->addr, m->sz & DMA_SZ_MASK); + } +} +#endif + int sx_cmdma_check(void) { uint32_t r = 0xFF; diff --git a/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/cmdma.h b/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/cmdma.h index 2e92a99e1b80..cd66cffe4b7e 100644 --- a/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/cmdma.h +++ b/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/cmdma.h @@ -28,7 +28,7 @@ #define DMATAG_CONFIG(offset) ((1 << 4) | (offset << 8)) /* can be 0, 1 or 2 */ -#define DMATAG_DATATYPE(x) (x << 6) +#define DMATAG_DATATYPE(x) (x << 6) #define DMATAG_DATATYPE_HEADER (1 << 6) #define DMATAG_DATATYPE_REFERENCE (3 << 6) @@ -106,17 +106,17 @@ (dmactl).d++; \ } while (0) -#define ADD_INDESC_BITS(dmactl, baddr, bsz, tag, msk, bitsz)\ - do {\ - size_t bitmask = (msk << 3) | 0x7;\ - size_t validbitsz = bitsz & bitmask;\ - if (validbitsz == 0)\ - validbitsz = bitmask + 1;\ - uint32_t asz = ALIGN_SZA(bsz, msk);\ - (dmactl).d->addr = sx_map_usrdatain((char *)(baddr), bsz);\ - (dmactl).d->sz = asz | DMA_REALIGN;\ - (dmactl).d->dmatag = tag | DMATAG_IGN((validbitsz - 1)); \ - (dmactl).d++;\ +#define ADD_INDESC_BITS(dmactl, baddr, bsz, tag, msk, bitsz) \ + do { \ + size_t bitmask = (msk << 3) | 0x7; \ + size_t validbitsz = bitsz & bitmask; \ + if (validbitsz == 0) \ + validbitsz = bitmask + 1; \ + uint32_t asz = ALIGN_SZA(bsz, msk); \ + (dmactl).d->addr = sx_map_usrdatain((char *)(baddr), bsz); \ + (dmactl).d->sz = asz | DMA_REALIGN; \ + (dmactl).d->dmatag = tag | DMATAG_IGN((validbitsz - 1)); \ + (dmactl).d++; \ } while (0) #define SET_LAST_DESC_IGN(dmactl, bsz, msk) \ @@ -163,6 +163,11 @@ void sx_cmdma_newcmd(struct sx_dmactl *dma, struct sxdesc *d, uint32_t cmd, uint /** Start input/fetcher DMA at indescs and output/pusher DMA at outdescs */ void sx_cmdma_start(struct sx_dmactl *dma, size_t privsz, struct sxdesc *indescs); +#ifdef CONFIG_DCACHE +/** Flush and invalidate the buffers for the output descriptors */ +void sx_cmdma_outdescs_flush_and_invd_dcache(const struct sx_dmactl *dma); +#endif + /** Return how the DMA is doing. * * Possible return values are: diff --git a/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/hash.c b/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/hash.c index 452f208ec59f..a8862f500101 100644 --- a/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/hash.c +++ b/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/hash.c @@ -235,7 +235,7 @@ static int start_hash_hw(struct sxhash *c) { sx_cmdma_start(&c->dma, sizeof(c->descs) + sizeof(c->extramem), c->descs); - return 0; + return SX_OK; } int sx_hash_save_state(struct sxhash *c) @@ -289,7 +289,8 @@ int sx_hash_status(struct sxhash *c) } #if CONFIG_DCACHE - sys_cache_data_invd_range((void *)&c->extramem, sizeof(c->extramem)); + sx_cmdma_outdescs_flush_and_invd_dcache(&c->dma); + sys_cache_data_flush_and_invd_range((void *)&c->extramem, sizeof(c->extramem)); #endif sx_hash_free(c); diff --git a/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/mac.c b/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/mac.c index 9bafbecbd15e..ca7b694c812a 100644 --- a/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/mac.c +++ b/subsys/nrf_security/src/drivers/cracen/sxsymcrypt/src/mac.c @@ -179,7 +179,8 @@ int sx_mac_status(struct sxmac *c) } #if CONFIG_DCACHE - sys_cache_data_invd_range((void *)&c->extramem, sizeof(c->extramem)); + sx_cmdma_outdescs_flush_and_invd_dcache(&c->dma); + sys_cache_data_flush_and_invd_range((void *)&c->extramem, sizeof(c->extramem)); #endif sx_mac_free(c);